diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..bc9b8b6f26678ef8bd4113e315d3956f7774720e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,29 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb filter=lfs diff=lfs merge=lfs -text +212118/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 filter=lfs diff=lfs merge=lfs -text +212118/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 filter=lfs diff=lfs merge=lfs -text +212118/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 filter=lfs diff=lfs merge=lfs -text +212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 filter=lfs diff=lfs merge=lfs -text +215240/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 filter=lfs diff=lfs merge=lfs -text +215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb filter=lfs diff=lfs merge=lfs -text +215240/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +215240/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +215240/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +215240/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag filter=lfs diff=lfs merge=lfs -text +212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag filter=lfs diff=lfs merge=lfs -text +215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag filter=lfs diff=lfs merge=lfs -text +215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag filter=lfs diff=lfs merge=lfs -text +215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag filter=lfs diff=lfs merge=lfs -text +215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag filter=lfs diff=lfs merge=lfs -text +212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag filter=lfs diff=lfs merge=lfs -text +215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag filter=lfs diff=lfs merge=lfs -text diff --git a/212117/.hydra/config.yaml b/212117/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7462739dbe7df1e4371890667e7c6210ec95aee --- /dev/null +++ b/212117/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '523_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/212117/.hydra/hydra.yaml b/212117/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1c696027c7e28c20dd6e4d2888ab1f05773eaee --- /dev/null +++ b/212117/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212117 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/212117/.hydra/overrides.yaml b/212117/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/212117/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/212117/eval_policy.log b/212117/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..e7c1e6c22c31e05890ef729dfd8663010dfd0898 --- /dev/null +++ b/212117/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-17 21:21:17,349][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-17 21:21:17,352][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-17 21:21:19,913][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/212118/.hydra/config.yaml b/212118/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7462739dbe7df1e4371890667e7c6210ec95aee --- /dev/null +++ b/212118/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '523_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/212118/.hydra/hydra.yaml b/212118/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74e09bb6e295c26275897eb4aa1e9045e40e70fe --- /dev/null +++ b/212118/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/212118/.hydra/overrides.yaml b/212118/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/212118/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/212118/episode_rosbags/aligned_depth_to_color_K.npy b/212118/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/212118/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/212118/episode_rosbags/cam_tf_world.npy b/212118/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/212118/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/212118/episode_rosbags/color_K.npy b/212118/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/212118/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/212118/episode_rosbags/depth_K.npy b/212118/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/212118/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag b/212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..5229cf50183787423b2eda328c7256d80a4a19ae --- /dev/null +++ b/212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39f43013b9eb9297f3fba753c7675143717f98c589ae81ac215d1d09aac5c6b9 +size 977351999 diff --git a/212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag b/212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..21151e5213c7c74ee93ff36608094c72a6cb8d81 --- /dev/null +++ b/212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18725042661a4db0597571d90cda790d55ec986b7265721a0e69644ae2239e3 +size 953326085 diff --git a/212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag b/212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..052923f5c2b6f3f243ea91560f8718b8e6788615 --- /dev/null +++ b/212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ce68c81d6da0c193b4e1c2cdc12989a0efcac454b23dc612fcae86932e72a50 +size 1416813567 diff --git a/212118/eval_robot.log b/212118/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..43137a60bf0d7219303c76ccecab6b506e1a8951 --- /dev/null +++ b/212118/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-17 21:21:18,583][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-17 21:21:18,586][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/212118/eval_video/0_eval.mp4 b/212118/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f21b30381e6d8c0a778f256aff8f278a1a5c47d5 --- /dev/null +++ b/212118/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17 +size 860246 diff --git a/212118/eval_video/1_eval.mp4 b/212118/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a6031d44888846474041b595bff9d1b0b54f460 --- /dev/null +++ b/212118/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3 +size 854722 diff --git a/212118/eval_video/2_eval.mp4 b/212118/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f41cf636677c538f8f7fa505e4d446df089cf1d --- /dev/null +++ b/212118/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318 +size 1236066 diff --git a/212118/eval_video/3_eval.mp4 b/212118/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/212118/eval_video/3_eval.mp4 differ diff --git a/212118/tb/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 b/212118/tb/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 new file mode 100644 index 0000000000000000000000000000000000000000..07b466d18f0423f30a201782998db96bfe060db2 --- /dev/null +++ b/212118/tb/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfdd281d85e27f83b5cfc0d25f514c1de66959b75befbd84590bc5f3ed7e29b5 +size 490 diff --git a/212118/wandb/debug-internal.log b/212118/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c8b1d6baf60954a5e7bbb795b5d5f3aa840b1771 --- /dev/null +++ b/212118/wandb/debug-internal.log @@ -0,0 +1,651 @@ +2024-12-17 21:21:24,431 INFO StreamThr :76461 [internal.py:wandb_internal():85] W&B internal server running at pid: 76461, started at: 2024-12-17 21:21:24.431534 +2024-12-17 21:21:24,433 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status +2024-12-17 21:21:24,444 INFO WriterThread:76461 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb +2024-12-17 21:21:24,444 DEBUG SenderThread:76461 [sender.py:send():379] send: header +2024-12-17 21:21:24,453 DEBUG SenderThread:76461 [sender.py:send():379] send: run +2024-12-17 21:21:24,691 INFO SenderThread:76461 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files +2024-12-17 21:21:24,691 INFO SenderThread:76461 [sender.py:_start_run_threads():1188] run started: 78r9r705 with start time 1734488484.432173 +2024-12-17 21:21:24,705 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: check_version +2024-12-17 21:21:24,706 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: check_version +2024-12-17 21:21:24,787 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: run_start +2024-12-17 21:21:24,792 DEBUG HandlerThread:76461 [system_info.py:__init__():26] System info init +2024-12-17 21:21:24,792 DEBUG HandlerThread:76461 [system_info.py:__init__():41] System info init done +2024-12-17 21:21:24,792 INFO HandlerThread:76461 [system_monitor.py:start():194] Starting system monitor +2024-12-17 21:21:24,792 INFO SystemMonitor:76461 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-17 21:21:24,792 INFO HandlerThread:76461 [system_monitor.py:probe():214] Collecting system info +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started cpu monitoring +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started disk monitoring +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started gpu monitoring +2024-12-17 21:21:24,794 INFO SystemMonitor:76461 [interfaces.py:start():188] Started memory monitoring +2024-12-17 21:21:24,794 INFO SystemMonitor:76461 [interfaces.py:start():188] Started network monitoring +2024-12-17 21:21:24,806 DEBUG HandlerThread:76461 [system_info.py:probe():152] Probing system +2024-12-17 21:21:24,807 DEBUG HandlerThread:76461 [system_info.py:_probe_git():137] Probing git +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:_probe_git():145] Probing git done +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:probe():200] Probing system done +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:21:24.806252', 'startedAt': '2024-12-18T02:21:24.423927', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.3751250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.58, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.929, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 389.73836517333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-17 21:21:24,812 INFO HandlerThread:76461 [system_monitor.py:probe():224] Finished collecting system info +2024-12-17 21:21:24,812 INFO HandlerThread:76461 [system_monitor.py:probe():227] Publishing system info +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:_save_code():44] Saving code +2024-12-17 21:21:24,814 DEBUG HandlerThread:76461 [system_info.py:_save_code():67] Saving code done +2024-12-17 21:21:24,814 DEBUG HandlerThread:76461 [system_info.py:_save_patches():84] Saving git patches +2024-12-17 21:21:24,883 DEBUG HandlerThread:76461 [system_info.py:_save_patches():126] Saving git patches done +2024-12-17 21:21:24,883 INFO HandlerThread:76461 [system_monitor.py:probe():229] Finished publishing system info +2024-12-17 21:21:24,886 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:21:24,886 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-17 21:21:24,887 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-17 21:21:24,887 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-17 21:21:25,126 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: python_packages +2024-12-17 21:21:25,126 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: python_packages +2024-12-17 21:21:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:25,128 DEBUG SenderThread:76461 [sender.py:send():379] send: telemetry +2024-12-17 21:21:25,133 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/zf5gfm67-diff.patch +2024-12-17 21:21:25,157 INFO wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/bk0z0q6f-code/FISH/eval_robot.py +2024-12-17 21:21:25,159 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/o7jx2utn-wandb-metadata.json +2024-12-17 21:21:25,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:25,169 INFO HandlerThread:76461 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb" + save: true +} +control { + relay_id: "52ec4a1767a3429cb6d1b8e86029eaaf" +} +_info { + stream_id: "78r9r705" +} + +2024-12-17 21:21:25,169 DEBUG HandlerThread:76461 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-17 21:21:25,170 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:25,277 DEBUG SenderThread:76461 [sender.py:send():379] send: tbrecord +2024-12-17 21:21:25,278 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:21:25,278 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 with policy live +2024-12-17 21:21:25,454 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lobsc1b3-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:25,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH +2024-12-17 21:21:25,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code +2024-12-17 21:21:26,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:27,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:27,692 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:28,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:29,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:30,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:30,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:31,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:31,696 DEBUG SenderThread:76461 [sender.py:send():379] send: config +2024-12-17 21:21:32,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:33,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:33,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:34,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:35,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:35,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:36,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:37,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:37,695 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:38,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:39,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:40,126 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:40,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:40,281 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:41,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:41,696 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:42,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:43,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:44,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:45,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:46,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:46,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:47,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:48,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:49,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:50,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:51,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:51,699 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:52,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:52,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:53,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:53,699 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:54,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:55,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:55,129 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:55,173 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:55,700 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:56,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:57,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:57,282 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:57,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml +2024-12-17 21:21:57,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:58,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:59,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:59,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:00,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:01,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:01,702 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:02,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:02,562 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:03,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:03,702 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:04,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:05,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:05,703 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:06,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:07,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:07,703 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:07,736 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:08,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:09,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:09,704 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:10,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:10,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:10,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:11,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:11,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:12,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:12,599 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:22:12,599 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 with policy now +2024-12-17 21:22:12,599 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:22:12,600 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:22:12,600 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:22:12,601 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos +2024-12-17 21:22:12,849 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/0wiijtor-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:13,108 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/2ltzk93y-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:13,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:13,611 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:13,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:14,138 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:15,134 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:16,134 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:17,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:17,707 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:18,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:18,625 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:19,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:19,707 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:20,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:21,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:22,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:23,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:24,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:24,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:24,794 DEBUG SystemMonitor:76461 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-17 21:22:24,795 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:22:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:25,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:25,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:26,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:27,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:28,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:29,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:29,217 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:30,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:31,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:32,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:33,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:34,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:35,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:35,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:36,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:37,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:37,713 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:22:37,978 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lzwjgswx-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:22:38,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:39,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:39,714 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:40,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:40,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:40,228 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:41,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:41,715 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:42,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:43,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:43,715 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:44,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:45,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:45,252 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:45,716 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:46,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:47,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:47,717 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:48,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:49,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:49,718 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:50,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:50,256 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:51,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:51,718 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:52,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:53,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:53,719 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:54,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:54,796 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:22:55,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:55,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:55,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:55,267 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:55,720 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:56,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:57,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:57,720 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:58,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:59,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:59,721 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:00,142 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:00,373 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:00,417 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:23:00,417 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:23:00,418 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 with policy now +2024-12-17 21:23:00,418 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:00,418 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:00,418 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:00,673 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/4e5seuc5-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:00,721 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:00,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:00,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:23:00,967 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/akortq61-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:01,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:01,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:02,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:03,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:04,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:05,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:05,723 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:06,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:06,160 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:07,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:07,724 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:08,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:09,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:10,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:10,131 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:10,173 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:11,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:11,232 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:12,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:13,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:14,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:15,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:16,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:17,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:17,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:18,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:19,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:20,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:21,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:21,571 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:21,571 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:21,571 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:22,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:23,029 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:23,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:23,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:24,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:24,796 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:23:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:25,128 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:25,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:25,730 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:26,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:27,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:27,731 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:28,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:28,194 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:29,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:29,731 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:30,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:31,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:31,732 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:32,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:33,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:33,393 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:33,733 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:34,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:35,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:35,734 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:36,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:37,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:37,734 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:38,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:38,405 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:39,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:39,735 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:40,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:40,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:41,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:41,736 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:42,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:43,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:43,409 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:43,737 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:44,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:45,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:45,737 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:46,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:47,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:47,738 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:48,151 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:48,460 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:49,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:49,739 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:50,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:51,151 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:51,740 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:52,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:52,900 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:23:52,900 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 with policy now +2024-12-17 21:23:52,900 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:23:52,901 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:52,901 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:52,901 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:53,155 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/evzpq5tf-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:53,155 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:23:53,920 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:54,016 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/jsf3d998-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:54,163 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:54,797 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:23:55,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:55,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:55,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:56,161 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:57,159 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:57,742 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:58,163 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:59,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:59,469 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:59,743 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:24:00,165 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:01,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:02,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:03,174 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:04,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:05,144 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:05,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:06,176 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:07,167 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:08,176 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:09,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:10,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:10,136 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:10,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:10,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:11,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:12,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:13,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:14,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:15,174 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:16,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:16,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:17,182 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:18,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:19,182 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:20,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:21,183 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:22,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:22,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:23,186 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:24,181 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:24,797 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:24:25,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:25,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:25,185 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:26,179 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:27,184 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:27,208 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:28,189 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:29,188 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:30,185 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:31,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:32,189 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:33,152 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:33,187 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:34,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:35,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:36,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:37,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:38,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:39,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:39,200 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:40,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:40,132 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:40,201 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:41,203 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:42,204 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:43,205 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:44,203 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:44,227 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:45,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:46,206 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:47,208 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:48,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:49,209 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:50,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:50,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:51,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:52,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:53,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:54,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:54,798 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:24:55,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:55,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:55,192 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:55,213 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:56,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:57,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:58,213 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:59,220 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:00,223 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:01,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:01,219 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:02,216 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:03,216 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:04,218 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:05,220 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:06,226 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:07,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:07,223 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:08,228 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:09,225 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:10,142 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:25:10,142 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:25:10,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:11,233 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:11,766 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:12,227 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:12,264 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:13,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:14,236 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:15,256 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:16,257 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:17,264 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:18,160 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:18,246 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:19,253 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:20,253 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:21,248 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:22,252 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:23,047 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:25:23,047 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now +2024-12-17 21:25:23,047 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:25:23,047 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:25:23,048 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:25:23,049 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:23,051 DEBUG SenderThread:76461 [sender.py:send():379] send: exit +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():586] handling exit code: 0 +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():588] handling runtime: 238 +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():594] send defer +2024-12-17 21:25:23,051 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,051 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-17 21:25:23,052 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 1 +2024-12-17 21:25:23,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-17 21:25:23,052 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 2 +2024-12-17 21:25:23,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [system_monitor.py:finish():203] Stopping system monitor +2024-12-17 21:25:23,052 DEBUG SystemMonitor:76461 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined cpu monitor +2024-12-17 21:25:23,052 DEBUG SystemMonitor:76461 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-17 21:25:23,053 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined disk monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined gpu monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined memory monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined network monitor +2024-12-17 21:25:23,096 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,096 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-17 21:25:23,096 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 3 +2024-12-17 21:25:23,097 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:25:23,097 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,097 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-17 21:25:23,097 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,097 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-17 21:25:23,097 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 4 +2024-12-17 21:25:23,097 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,097 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-17 21:25:23,269 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/rrl4ps4z-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:23,771 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:25:24,015 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/ipbifp7e-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:28,586 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:28,586 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:28,586 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,587 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-17 21:25:28,587 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 5 +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:28,587 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:25:28,587 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:28,588 DEBUG SenderThread:76461 [sender.py:send():379] send: summary +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:28,588 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 6 +2024-12-17 21:25:28,588 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,588 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-17 21:25:28,589 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 7 +2024-12-17 21:25:28,589 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:28,589 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,589 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-17 21:25:28,589 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-17 21:25:28,772 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:29,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: keepalive +2024-12-17 21:25:29,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:29,669 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 8 +2024-12-17 21:25:29,669 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:29,669 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:29,669 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-17 21:25:29,670 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:29,670 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-17 21:25:29,670 INFO SenderThread:76461 [job_builder.py:build():440] Attempting to build job artifact +2024-12-17 21:25:29,672 INFO SenderThread:76461 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-17 21:25:29,686 INFO SenderThread:76461 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-17 21:25:29,700 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 9 +2024-12-17 21:25:29,700 DEBUG SenderThread:76461 [sender.py:send():379] send: artifact +2024-12-17 21:25:29,700 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:29,702 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-17 21:25:29,772 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:30,053 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:30,189 INFO wandb-upload_1:76461 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpat7ok6vh +2024-12-17 21:25:30,320 INFO wandb-upload_2:76461 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu793frvf +2024-12-17 21:25:30,360 INFO wandb-upload_0:76461 [upload_job.py:push():88] Uploaded file /tmp/tmp41i79nsx/wandb-job.json +2024-12-17 21:25:30,913 INFO SenderThread:76461 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'versionIndex': 378}}} +2024-12-17 21:25:30,913 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:30,913 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-17 21:25:30,913 INFO SenderThread:76461 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml config.yaml +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json wandb-summary.json +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt requirements.txt +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json wandb-metadata.json +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log output.log +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch diff.patch +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:25:31,778 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:25:31,779 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:25:31,780 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 10 +2024-12-17 21:25:31,780 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:31,782 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:31,782 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-17 21:25:31,782 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:31,782 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-17 21:25:31,782 INFO SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:25:31,949 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:31,988 INFO wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml +2024-12-17 21:25:31,990 INFO wandb-upload_3:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt +2024-12-17 21:25:32,014 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/kg0vdiiy-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:25:32,036 INFO wandb-upload_4:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:32,053 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,054 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,237 INFO Thread-11 (_thread_body):76461 [sender.py:transition_state():614] send defer: 11 +2024-12-17 21:25:32,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,237 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-17 21:25:32,237 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,237 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-17 21:25:32,237 INFO SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher +2024-12-17 21:25:32,238 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 12 +2024-12-17 21:25:32,238 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,238 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-17 21:25:32,238 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,238 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-17 21:25:32,238 INFO SenderThread:76461 [file_stream.py:finish():601] file stream finish called +2024-12-17 21:25:32,430 INFO SenderThread:76461 [file_stream.py:finish():605] file stream finish is done +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 13 +2024-12-17 21:25:32,430 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,430 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-17 21:25:32,430 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 14 +2024-12-17 21:25:32,430 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,431 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send():379] send: final +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send():379] send: footer +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,431 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-17 21:25:32,431 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,431 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: server_info +2024-12-17 21:25:32,432 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,432 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,432 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: server_info +2024-12-17 21:25:32,432 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: get_summary +2024-12-17 21:25:32,434 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-17 21:25:32,434 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:32,507 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-17 21:25:32,507 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: shutdown +2024-12-17 21:25:32,507 INFO HandlerThread:76461 [handler.py:finish():882] shutting down handler +2024-12-17 21:25:33,432 INFO WriterThread:76461 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb +2024-12-17 21:25:33,507 INFO SenderThread:76461 [sender.py:finish():1615] shutting down sender +2024-12-17 21:25:33,507 INFO SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:25:33,507 INFO SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher diff --git a/212118/wandb/debug.log b/212118/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..769b44f57a072605182fa99537bc3e7590c8d14c --- /dev/null +++ b/212118/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Configure stats pid to 76272 +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/settings +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():569] calling init triggers +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118'} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():619] starting backend +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():623] setting up manager +2024-12-17 21:21:24,430 INFO MainThread:76272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-17 21:21:24,431 INFO MainThread:76272 [wandb_init.py:init():631] backend started and connected +2024-12-17 21:21:24,443 INFO MainThread:76272 [wandb_init.py:init():720] updated telemetry +2024-12-17 21:21:24,450 INFO MainThread:76272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-17 21:21:24,705 INFO MainThread:76272 [wandb_run.py:_on_init():2435] communicating current version +2024-12-17 21:21:24,776 INFO MainThread:76272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-17 21:21:24,779 INFO MainThread:76272 [wandb_init.py:init():804] starting run threads in backend +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_console_start():2413] atexit reg +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-17 21:21:25,129 INFO MainThread:76272 [wandb_init.py:init():847] run started, returning control to user process +2024-12-17 21:21:25,129 INFO MainThread:76272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb, True +2024-12-17 21:21:31,696 INFO MainThread:76272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2024-12-17 21:25:33,510 WARNING MsgRouterThr:76272 [router.py:message_loop():77] message_loop has been closed diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py b/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml b/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e66ae04a1b6e83a504130196a854e45151b4308f --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml @@ -0,0 +1,891 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '523_0' +wandb_notes: + desc: null + value: 523_0_req_603_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - fowlers_twodim +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1734488484 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch b/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..07ef51bba00eb07cad49fbccb9c4c98ad2796b36 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch @@ -0,0 +1,195 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb0a413..6b5d639 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -107,7 +107,7 @@ num_valid_demos: null + val_num_groups: 3 + + # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act' +-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act' ++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act' + expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}' + + expert_dataset: '${expert_dataset_dirpath}/demos.zarr' +@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} +-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full 32 demos +-# wandb_run_id: "265625" # RGBD history seed 0 +-# wandb_run_id: "266370_1" # RGBD history seed 1 +-# wandb_run_id: "266370_2" # RGBD history seed 2 +-# wandb_run_id: "266382_4" # RGBD history seed 4 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0 +-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1 +-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2 +- +-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0 +-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1 +-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2 +-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3 +-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4 +- +-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0 +-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1 +-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2 +-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3 +-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 +- +-# 8 held out demos (26 demos) +-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0 +-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1 +-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2 +- +-# 8 held out demos (26 demos) +-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0 +-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1 +-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0 +-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1 +-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2 +-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0 +-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1 +-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2 +-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4 +- +- +- +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full dataset +-# 6 step action history +-# wandb_run_id: '250358_0' # RGBD+act history +-# wandb_run_id: '250371_0' # RGBD+mask+act history +-# wandb_run_id: '250366_0' # RGBD+contact+act history +-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250366_0' # RGBD+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 16 held out demos +-# wandb_run_id: '250362_0' # RGBD+act history +-# wandb_run_id: '250373_0' # RGBD+mask+act history +-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250368_0' # RGBD+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 32 held out demos +-# wandb_run_id: '250365_0' # RGBD+act history +-# wandb_run_id: '250375_0' # RGBD+mask+act history +-wandb_run_id: '250380_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250370_0' # RGBD+contact+act history +-# 4 step action history +-# wandb_run_id: '264772_0' # RGBD+act history +-# wandb_run_id: '264775_0' # RGBD+mask+act history +-# wandb_run_id: '264776_0' # RGBD+contact+act history +-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 16 held out demos +-# wandb_run_id: '250362_0' # RGBD+act history +-# wandb_run_id: '250373_0' # RGBD+mask+act history +-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250368_0' # RGBD+contact+act history ++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# 14 demos ++ ++# RGBD+act history ++# wandb_run_id: '497_0' # german ++# wandb_run_id: '505_0' # hbm ++# wandb_run_id: '506_0' # lady ++# wandb_run_id: '507_0' # greece ++# wandb_run_id: '508_0' # fowlers ++# wandb_run_id: '509_0' # lib ++# wandb_run_id: '510_0' # modelsys ++# wandb_run_id: '511_0' # electrodyn ++ ++# RGBD+mask+act history ++# wandb_run_id: '512_0' # german ++# wandb_run_id: '520_0' # hbm ++# wandb_run_id: '521_0' # lady ++# wandb_run_id: '522_0' # greece ++wandb_run_id: '523_0' # fowlers ++# wandb_run_id: '524_0' # lib ++# wandb_run_id: '525_0' # modelsys ++# wandb_run_id: '526_0' # electrodyn + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 32 held out demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 9406a46..0f225ec 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists(): + local_checkpoint_root_dir.mkdir(parents=True) + + # Download the model checkpoints +-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') + remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser() + remote_username = 'leonmkim' +-remote_host = 'halley' ++remote_host = 'grasp-login1' + + #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 ++# run_id_list = [ ++# '512_0', # german ++# '520_0', # hbm ++# '521_0', # lady ++# '522_0', # greece ++# '523_0', # fowlers ++# '524_0', # lib ++# '525_0', # modelsys ++# '526_0', # electrodyn ++# ] ++ + # run_id_list = [ + # '265638', + # '265637', +@@ -75,6 +87,17 @@ remote_host = 'halley' + # ] + + # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 ++# run_id_list = [ ++# '497_0', # german ++# '505_0', # hbm ++# '506_0', # lady ++# '507_0', # greece ++# '508_0', # fowlers ++# '509_0', # lib ++# '510_0', # modelsys ++# '511_0', # electrodyn ++# ] ++ + # run_id_list = [ + # '266382_4', + # '266370_2', +@@ -116,13 +139,13 @@ remote_host = 'halley' + # ] + + # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +-run_id_list = [ +- # '266384_4,# not yet available +- '266372_1', +- '266372_2', +- '265627', +- '265620', +-] ++# run_id_list = [ ++# # '266384_4,# not yet available ++# '266372_1', ++# '266372_2', ++# '265627', ++# '265620', ++# ] + checkpoint_epoch = 12000 + + # use subprocess to download the model checkpoints in parallel diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f21b30381e6d8c0a778f256aff8f278a1a5c47d5 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17 +size 860246 diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a6031d44888846474041b595bff9d1b0b54f460 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3 +size 854722 diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2f41cf636677c538f8f7fa505e4d446df089cf1d --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318 +size 1236066 diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 differ diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/output.log b/212118/wandb/run-20241217_212124-78r9r705/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3727edf2730166224e7f111b6cbf8ec6d555762c --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/output.log @@ -0,0 +1,3004 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1734488491.696665]: resetting environment +[INFO] [1734488491.699516]: cleared current plan +[INFO] [1734488491.700458]: moving to home +[INFO] [1734488492.702628]: reached home +[INFO] [1734488492.704727]: reset action history +[INFO] [1734488495.414691]: environment reset +Starting episode 0 +[INFO] [1734488495.416885]: resetting environment +[INFO] [1734488495.420430]: cleared current plan +[INFO] [1734488495.421459]: moving to home +[INFO] [1734488496.424063]: reached home +[INFO] [1734488496.425417]: reset action history +[INFO] [1734488499.137813]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734488509.040145]: resetting environment +[INFO] [1734488509.043179]: cleared current plan +[INFO] [1734488509.044012]: moving to home +[INFO] [1734488510.045332]: reached home +[INFO] [1734488510.046555]: reset action history +[INFO] [1734488512.753081]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_0_2024-12-17-21-21-52.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log +started roslaunch server http://158.130.50.37:42227/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [76587] +process[print_text-2]: started with pid [76588] +process[rosbag_record-3]: started with pid [76589] +started bagging! +[INFO] [1734488513.535930]: received mask_obs with header seq: 1330 +stamp: + secs: 1734488513 + nsecs: 540523290 +frame_id: '' and grasped_object_mask header seq: 117429 +stamp: + secs: 1734488513 + nsecs: 306998968 +frame_id: "camera_color_optical_frame" +[INFO] [1734488513.720297]: received mask_obs with header seq: 1331 +stamp: + secs: 1734488513 + nsecs: 725090742 +frame_id: '' and grasped_object_mask header seq: 117435 +stamp: + secs: 1734488513 + nsecs: 506836653 +frame_id: "camera_color_optical_frame" +[INFO] [1734488514.117301]: received mask_obs with header seq: 1332 +stamp: + secs: 1734488514 + nsecs: 121912002 +frame_id: '' and grasped_object_mask header seq: 117447 +stamp: + secs: 1734488513 + nsecs: 906508684 +frame_id: "camera_color_optical_frame" +[INFO] [1734488514.291310]: received mask_obs with header seq: 1333 +stamp: + secs: 1734488514 + nsecs: 296135663 +frame_id: '' and grasped_object_mask header seq: 117452 +stamp: + secs: 1734488514 + nsecs: 73037863 +frame_id: "camera_color_optical_frame" +[INFO] [1734488514.494223]: received mask_obs with header seq: 1334 +stamp: + secs: 1734488514 + nsecs: 498311281 +frame_id: '' and grasped_object_mask header seq: 117457 +stamp: + secs: 1734488514 + nsecs: 239567757 +frame_id: "camera_color_optical_frame" +[INFO] [1734488514.681076]: received mask_obs with header seq: 1335 +stamp: + secs: 1734488514 + nsecs: 685832262 +frame_id: '' and grasped_object_mask header seq: 117463 +stamp: + secs: 1734488514 + nsecs: 439402819 +frame_id: "camera_color_optical_frame" +[INFO] [1734488514.870616]: received mask_obs with header seq: 1336 +stamp: + secs: 1734488514 + nsecs: 874481916 +frame_id: '' and grasped_object_mask header seq: 117469 +stamp: + secs: 1734488514 + nsecs: 640603304 +frame_id: "camera_color_optical_frame" +[INFO] [1734488515.073492]: received mask_obs with header seq: 1337 +stamp: + secs: 1734488515 + nsecs: 77445268 +frame_id: '' and grasped_object_mask header seq: 117475 +stamp: + secs: 1734488514 + nsecs: 842766285 +frame_id: "camera_color_optical_frame" +[INFO] [1734488515.266401]: received mask_obs with header seq: 1338 +stamp: + secs: 1734488515 + nsecs: 270193815 +frame_id: '' and grasped_object_mask header seq: 117480 +stamp: + secs: 1734488515 + nsecs: 11301756 +frame_id: "camera_color_optical_frame" +[INFO] [1734488515.474418]: received mask_obs with header seq: 1339 +stamp: + secs: 1734488515 + nsecs: 479188203 +frame_id: '' and grasped_object_mask header seq: 117486 +stamp: + secs: 1734488515 + nsecs: 213622570 +frame_id: "camera_color_optical_frame" +[INFO] [1734488515.676012]: received mask_obs with header seq: 1340 +stamp: + secs: 1734488515 + nsecs: 680783033 +frame_id: '' and grasped_object_mask header seq: 117493 +stamp: + secs: 1734488515 + nsecs: 449772596 +frame_id: "camera_color_optical_frame" +[INFO] [1734488515.874816]: received mask_obs with header seq: 1341 +stamp: + secs: 1734488515 + nsecs: 879640102 +frame_id: '' and grasped_object_mask header seq: 117499 +stamp: + secs: 1734488515 + nsecs: 650627136 +frame_id: "camera_color_optical_frame" +[INFO] [1734488516.073933]: received mask_obs with header seq: 1342 +stamp: + secs: 1734488516 + nsecs: 78693151 +frame_id: '' and grasped_object_mask header seq: 117505 +stamp: + secs: 1734488515 + nsecs: 850232840 +frame_id: "camera_color_optical_frame" +[INFO] [1734488516.271909]: received mask_obs with header seq: 1343 +stamp: + secs: 1734488516 + nsecs: 275898456 +frame_id: '' and grasped_object_mask header seq: 117511 +stamp: + secs: 1734488516 + nsecs: 49826384 +frame_id: "camera_color_optical_frame" +[INFO] [1734488516.467535]: received mask_obs with header seq: 1344 +stamp: + secs: 1734488516 + nsecs: 471306562 +frame_id: '' and grasped_object_mask header seq: 117516 +stamp: + secs: 1734488516 + nsecs: 216145992 +frame_id: "camera_color_optical_frame" +[INFO] [1734488516.672146]: received mask_obs with header seq: 1345 +stamp: + secs: 1734488516 + nsecs: 676180362 +frame_id: '' and grasped_object_mask header seq: 117522 +stamp: + secs: 1734488516 + nsecs: 415718079 +frame_id: "camera_color_optical_frame" +[INFO] [1734488516.875914]: received mask_obs with header seq: 1346 +stamp: + secs: 1734488516 + nsecs: 880681514 +frame_id: '' and grasped_object_mask header seq: 117528 +stamp: + secs: 1734488516 + nsecs: 615297556 +frame_id: "camera_color_optical_frame" +[INFO] [1734488517.081978]: received mask_obs with header seq: 1347 +stamp: + secs: 1734488517 + nsecs: 86817026 +frame_id: '' and grasped_object_mask header seq: 117535 +stamp: + secs: 1734488516 + nsecs: 848172426 +frame_id: "camera_color_optical_frame" +[INFO] [1734488517.279205]: received mask_obs with header seq: 1348 +stamp: + secs: 1734488517 + nsecs: 283457756 +frame_id: '' and grasped_object_mask header seq: 117541 +stamp: + secs: 1734488517 + nsecs: 47765970 +frame_id: "camera_color_optical_frame" +[INFO] [1734488517.476432]: received mask_obs with header seq: 1349 +stamp: + secs: 1734488517 + nsecs: 481223821 +frame_id: '' and grasped_object_mask header seq: 117547 +stamp: + secs: 1734488517 + nsecs: 247347593 +frame_id: "camera_color_optical_frame" +[INFO] [1734488517.673335]: received mask_obs with header seq: 1350 +stamp: + secs: 1734488517 + nsecs: 678065299 +frame_id: '' and grasped_object_mask header seq: 117553 +stamp: + secs: 1734488517 + nsecs: 446918726 +frame_id: "camera_color_optical_frame" +[INFO] [1734488517.871731]: received mask_obs with header seq: 1351 +stamp: + secs: 1734488517 + nsecs: 876430988 +frame_id: '' and grasped_object_mask header seq: 117559 +stamp: + secs: 1734488517 + nsecs: 646504879 +frame_id: "camera_color_optical_frame" +[INFO] [1734488518.076064]: received mask_obs with header seq: 1352 +stamp: + secs: 1734488518 + nsecs: 80809831 +frame_id: '' and grasped_object_mask header seq: 117564 +stamp: + secs: 1734488517 + nsecs: 812840939 +frame_id: "camera_color_optical_frame" +[INFO] [1734488518.276494]: received mask_obs with header seq: 1353 +stamp: + secs: 1734488518 + nsecs: 281139850 +frame_id: '' and grasped_object_mask header seq: 117571 +stamp: + secs: 1734488518 + nsecs: 45698166 +frame_id: "camera_color_optical_frame" +[INFO] [1734488518.475418]: received mask_obs with header seq: 1354 +stamp: + secs: 1734488518 + nsecs: 480235338 +frame_id: '' and grasped_object_mask header seq: 117577 +stamp: + secs: 1734488518 + nsecs: 245277166 +frame_id: "camera_color_optical_frame" +[INFO] [1734488518.670695]: received mask_obs with header seq: 1355 +stamp: + secs: 1734488518 + nsecs: 675471067 +frame_id: '' and grasped_object_mask header seq: 117583 +stamp: + secs: 1734488518 + nsecs: 444844007 +frame_id: "camera_color_optical_frame" +[INFO] [1734488518.858743]: received mask_obs with header seq: 1356 +stamp: + secs: 1734488518 + nsecs: 863535881 +frame_id: '' and grasped_object_mask header seq: 117588 +stamp: + secs: 1734488518 + nsecs: 611164093 +frame_id: "camera_color_optical_frame" +[INFO] [1734488519.050615]: received mask_obs with header seq: 1357 +stamp: + secs: 1734488519 + nsecs: 55445671 +frame_id: '' and grasped_object_mask header seq: 117594 +stamp: + secs: 1734488518 + nsecs: 810777903 +frame_id: "camera_color_optical_frame" +[INFO] [1734488519.245232]: received mask_obs with header seq: 1358 +stamp: + secs: 1734488519 + nsecs: 249953031 +frame_id: '' and grasped_object_mask header seq: 117600 +stamp: + secs: 1734488519 + nsecs: 10380030 +frame_id: "camera_color_optical_frame" +[INFO] [1734488519.442395]: received mask_obs with header seq: 1359 +stamp: + secs: 1734488519 + nsecs: 447017192 +frame_id: '' and grasped_object_mask header seq: 117606 +stamp: + secs: 1734488519 + nsecs: 209971666 +frame_id: "camera_color_optical_frame" +[INFO] [1734488519.640498]: received mask_obs with header seq: 1360 +stamp: + secs: 1734488519 + nsecs: 645313978 +frame_id: '' and grasped_object_mask header seq: 117612 +stamp: + secs: 1734488519 + nsecs: 409551144 +frame_id: "camera_color_optical_frame" +[INFO] [1734488519.837375]: received mask_obs with header seq: 1361 +stamp: + secs: 1734488519 + nsecs: 842097043 +frame_id: '' and grasped_object_mask header seq: 117618 +stamp: + secs: 1734488519 + nsecs: 609138727 +frame_id: "camera_color_optical_frame" +[INFO] [1734488520.037435]: received mask_obs with header seq: 1362 +stamp: + secs: 1734488520 + nsecs: 41568517 +frame_id: '' and grasped_object_mask header seq: 117624 +stamp: + secs: 1734488519 + nsecs: 808754683 +frame_id: "camera_color_optical_frame" +[INFO] [1734488520.230878]: received mask_obs with header seq: 1363 +stamp: + secs: 1734488520 + nsecs: 235653162 +frame_id: '' and grasped_object_mask header seq: 117629 +stamp: + secs: 1734488519 + nsecs: 975093365 +frame_id: "camera_color_optical_frame" +[INFO] [1734488520.417156]: received mask_obs with header seq: 1364 +stamp: + secs: 1734488520 + nsecs: 421958446 +frame_id: '' and grasped_object_mask header seq: 117635 +stamp: + secs: 1734488520 + nsecs: 174688816 +frame_id: "camera_color_optical_frame" +[INFO] [1734488520.613280]: received mask_obs with header seq: 1365 +stamp: + secs: 1734488520 + nsecs: 617956876 +frame_id: '' and grasped_object_mask header seq: 117641 +stamp: + secs: 1734488520 + nsecs: 374273062 +frame_id: "camera_color_optical_frame" +[INFO] [1734488520.806447]: received mask_obs with header seq: 1366 +stamp: + secs: 1734488520 + nsecs: 811276197 +frame_id: '' and grasped_object_mask header seq: 117647 +stamp: + secs: 1734488520 + nsecs: 573854208 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.005897]: received mask_obs with header seq: 1367 +stamp: + secs: 1734488521 + nsecs: 10637044 +frame_id: '' and grasped_object_mask header seq: 117653 +stamp: + secs: 1734488520 + nsecs: 773466349 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.203162]: received mask_obs with header seq: 1368 +stamp: + secs: 1734488521 + nsecs: 207555055 +frame_id: '' and grasped_object_mask header seq: 117659 +stamp: + secs: 1734488520 + nsecs: 973066807 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.393816]: received mask_obs with header seq: 1369 +stamp: + secs: 1734488521 + nsecs: 398211240 +frame_id: '' and grasped_object_mask header seq: 117664 +stamp: + secs: 1734488521 + nsecs: 139392614 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.583850]: received mask_obs with header seq: 1370 +stamp: + secs: 1734488521 + nsecs: 588620901 +frame_id: '' and grasped_object_mask header seq: 117670 +stamp: + secs: 1734488521 + nsecs: 338971853 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.782108]: received mask_obs with header seq: 1371 +stamp: + secs: 1734488521 + nsecs: 786865711 +frame_id: '' and grasped_object_mask header seq: 117676 +stamp: + secs: 1734488521 + nsecs: 538539886 +frame_id: "camera_color_optical_frame" +[INFO] [1734488521.975370]: received mask_obs with header seq: 1372 +stamp: + secs: 1734488521 + nsecs: 980169534 +frame_id: '' and grasped_object_mask header seq: 117682 +stamp: + secs: 1734488521 + nsecs: 738153458 +frame_id: "camera_color_optical_frame" +[INFO] [1734488522.173034]: received mask_obs with header seq: 1373 +stamp: + secs: 1734488522 + nsecs: 177629947 +frame_id: '' and grasped_object_mask header seq: 117688 +stamp: + secs: 1734488521 + nsecs: 937754631 +frame_id: "camera_color_optical_frame" +[INFO] [1734488522.373051]: received mask_obs with header seq: 1374 +stamp: + secs: 1734488522 + nsecs: 377842187 +frame_id: '' and grasped_object_mask header seq: 117694 +stamp: + secs: 1734488522 + nsecs: 137343645 +frame_id: "camera_color_optical_frame" +[INFO] [1734488522.562010]: received mask_obs with header seq: 1375 +stamp: + secs: 1734488522 + nsecs: 566809654 +frame_id: '' and grasped_object_mask header seq: 117699 +stamp: + secs: 1734488522 + nsecs: 303659678 +frame_id: "camera_color_optical_frame" +[INFO] [1734488522.760386]: received mask_obs with header seq: 1376 +stamp: + secs: 1734488522 + nsecs: 765190124 +frame_id: '' and grasped_object_mask header seq: 117706 +stamp: + secs: 1734488522 + nsecs: 536486864 +frame_id: "camera_color_optical_frame" +[INFO] [1734488522.959358]: received mask_obs with header seq: 1377 +stamp: + secs: 1734488522 + nsecs: 963238239 +frame_id: '' and grasped_object_mask header seq: 117712 +stamp: + secs: 1734488522 + nsecs: 736104250 +frame_id: "camera_color_optical_frame" +[INFO] [1734488523.166117]: received mask_obs with header seq: 1378 +stamp: + secs: 1734488523 + nsecs: 170861482 +frame_id: '' and grasped_object_mask header seq: 117718 +stamp: + secs: 1734488522 + nsecs: 935710907 +frame_id: "camera_color_optical_frame" +[INFO] [1734488523.363388]: received mask_obs with header seq: 1379 +stamp: + secs: 1734488523 + nsecs: 368171215 +frame_id: '' and grasped_object_mask header seq: 117724 +stamp: + secs: 1734488523 + nsecs: 135305405 +frame_id: "camera_color_optical_frame" +[INFO] [1734488523.563067]: received mask_obs with header seq: 1380 +stamp: + secs: 1734488523 + nsecs: 567840337 +frame_id: '' and grasped_object_mask header seq: 117730 +stamp: + secs: 1734488523 + nsecs: 334888458 +frame_id: "camera_color_optical_frame" +[INFO] [1734488523.760427]: received mask_obs with header seq: 1381 +stamp: + secs: 1734488523 + nsecs: 764970064 +frame_id: '' and grasped_object_mask header seq: 117736 +stamp: + secs: 1734488523 + nsecs: 534459829 +frame_id: "camera_color_optical_frame" +[INFO] [1734488523.957353]: received mask_obs with header seq: 1382 +stamp: + secs: 1734488523 + nsecs: 962140083 +frame_id: '' and grasped_object_mask header seq: 117742 +stamp: + secs: 1734488523 + nsecs: 734083652 +frame_id: "camera_color_optical_frame" +[INFO] [1734488524.152965]: received mask_obs with header seq: 1383 +stamp: + secs: 1734488524 + nsecs: 157132863 +frame_id: '' and grasped_object_mask header seq: 117747 +stamp: + secs: 1734488523 + nsecs: 900429010 +frame_id: "camera_color_optical_frame" +[INFO] [1734488524.342782]: received mask_obs with header seq: 1384 +stamp: + secs: 1734488524 + nsecs: 347471475 +frame_id: '' and grasped_object_mask header seq: 117753 +stamp: + secs: 1734488524 + nsecs: 100032806 +frame_id: "camera_color_optical_frame" +[INFO] [1734488524.537917]: received mask_obs with header seq: 1385 +stamp: + secs: 1734488524 + nsecs: 542565584 +frame_id: '' and grasped_object_mask header seq: 117759 +stamp: + secs: 1734488524 + nsecs: 299625397 +frame_id: "camera_color_optical_frame" +[INFO] [1734488524.737870]: received mask_obs with header seq: 1386 +stamp: + secs: 1734488524 + nsecs: 742503643 +frame_id: '' and grasped_object_mask header seq: 117765 +stamp: + secs: 1734488524 + nsecs: 499207258 +frame_id: "camera_color_optical_frame" +[INFO] [1734488524.938627]: received mask_obs with header seq: 1387 +stamp: + secs: 1734488524 + nsecs: 943261861 +frame_id: '' and grasped_object_mask header seq: 117771 +stamp: + secs: 1734488524 + nsecs: 698829889 +frame_id: "camera_color_optical_frame" +[INFO] [1734488525.151322]: received mask_obs with header seq: 1388 +stamp: + secs: 1734488525 + nsecs: 156005382 +frame_id: '' and grasped_object_mask header seq: 117777 +stamp: + secs: 1734488524 + nsecs: 898440123 +frame_id: "camera_color_optical_frame" +[INFO] [1734488525.343674]: received mask_obs with header seq: 1389 +stamp: + secs: 1734488525 + nsecs: 348135948 +frame_id: '' and grasped_object_mask header seq: 117783 +stamp: + secs: 1734488525 + nsecs: 98038435 +frame_id: "camera_color_optical_frame" +[INFO] [1734488525.540274]: received mask_obs with header seq: 1390 +stamp: + secs: 1734488525 + nsecs: 545060396 +frame_id: '' and grasped_object_mask header seq: 117789 +stamp: + secs: 1734488525 + nsecs: 297625065 +frame_id: "camera_color_optical_frame" +[INFO] [1734488525.737348]: received mask_obs with header seq: 1391 +stamp: + secs: 1734488525 + nsecs: 741950035 +frame_id: '' and grasped_object_mask header seq: 117795 +stamp: + secs: 1734488525 + nsecs: 497200489 +frame_id: "camera_color_optical_frame" +[INFO] [1734488525.950629]: received mask_obs with header seq: 1392 +stamp: + secs: 1734488525 + nsecs: 955370903 +frame_id: '' and grasped_object_mask header seq: 117801 +stamp: + secs: 1734488525 + nsecs: 696797848 +frame_id: "camera_color_optical_frame" +[INFO] [1734488526.139287]: received mask_obs with header seq: 1393 +stamp: + secs: 1734488526 + nsecs: 143530845 +frame_id: '' and grasped_object_mask header seq: 117807 +stamp: + secs: 1734488525 + nsecs: 896395445 +frame_id: "camera_color_optical_frame" +[INFO] [1734488526.354287]: received mask_obs with header seq: 1394 +stamp: + secs: 1734488526 + nsecs: 359120130 +frame_id: '' and grasped_object_mask header seq: 117813 +stamp: + secs: 1734488526 + nsecs: 95981836 +frame_id: "camera_color_optical_frame" +[INFO] [1734488526.544801]: received mask_obs with header seq: 1395 +stamp: + secs: 1734488526 + nsecs: 549009561 +frame_id: '' and grasped_object_mask header seq: 117819 +stamp: + secs: 1734488526 + nsecs: 295555353 +frame_id: "camera_color_optical_frame" +[INFO] [1734488526.737395]: received mask_obs with header seq: 1396 +stamp: + secs: 1734488526 + nsecs: 742144584 +frame_id: '' and grasped_object_mask header seq: 117825 +stamp: + secs: 1734488526 + nsecs: 495116949 +frame_id: "camera_color_optical_frame" +[INFO] [1734488526.937526]: received mask_obs with header seq: 1397 +stamp: + secs: 1734488526 + nsecs: 941965341 +frame_id: '' and grasped_object_mask header seq: 117831 +stamp: + secs: 1734488526 + nsecs: 694713354 +frame_id: "camera_color_optical_frame" +[INFO] [1734488527.135987]: received mask_obs with header seq: 1398 +stamp: + secs: 1734488527 + nsecs: 140719413 +frame_id: '' and grasped_object_mask header seq: 117837 +stamp: + secs: 1734488526 + nsecs: 894314528 +frame_id: "camera_color_optical_frame" +[INFO] [1734488527.337024]: received mask_obs with header seq: 1399 +stamp: + secs: 1734488527 + nsecs: 341829061 +frame_id: '' and grasped_object_mask header seq: 117843 +stamp: + secs: 1734488527 + nsecs: 93903780 +frame_id: "camera_color_optical_frame" +[INFO] [1734488527.536707]: received mask_obs with header seq: 1400 +stamp: + secs: 1734488527 + nsecs: 541493177 +frame_id: '' and grasped_object_mask header seq: 117849 +stamp: + secs: 1734488527 + nsecs: 293482304 +frame_id: "camera_color_optical_frame" +[INFO] [1734488527.735802]: received mask_obs with header seq: 1401 +stamp: + secs: 1734488527 + nsecs: 740492105 +frame_id: '' and grasped_object_mask header seq: 117855 +stamp: + secs: 1734488527 + nsecs: 493048191 +frame_id: "camera_color_optical_frame" +[INFO] [1734488527.936937]: received mask_obs with header seq: 1402 +stamp: + secs: 1734488527 + nsecs: 941634416 +frame_id: '' and grasped_object_mask header seq: 117861 +stamp: + secs: 1734488527 + nsecs: 692643404 +frame_id: "camera_color_optical_frame" +[INFO] [1734488528.136992]: received mask_obs with header seq: 1403 +stamp: + secs: 1734488528 + nsecs: 141670703 +frame_id: '' and grasped_object_mask header seq: 117867 +stamp: + secs: 1734488527 + nsecs: 892241240 +frame_id: "camera_color_optical_frame" +[INFO] [1734488528.336021]: received mask_obs with header seq: 1404 +stamp: + secs: 1734488528 + nsecs: 340800285 +frame_id: '' and grasped_object_mask header seq: 117873 +stamp: + secs: 1734488528 + nsecs: 91827154 +frame_id: "camera_color_optical_frame" +[INFO] [1734488528.536392]: received mask_obs with header seq: 1405 +stamp: + secs: 1734488528 + nsecs: 541168689 +frame_id: '' and grasped_object_mask header seq: 117879 +stamp: + secs: 1734488528 + nsecs: 291401148 +frame_id: "camera_color_optical_frame" +[INFO] [1734488528.740782]: received mask_obs with header seq: 1406 +stamp: + secs: 1734488528 + nsecs: 745445489 +frame_id: '' and grasped_object_mask header seq: 117885 +stamp: + secs: 1734488528 + nsecs: 490964174 +frame_id: "camera_color_optical_frame" +[INFO] [1734488528.937434]: received mask_obs with header seq: 1407 +stamp: + secs: 1734488528 + nsecs: 942167997 +frame_id: '' and grasped_object_mask header seq: 117891 +stamp: + secs: 1734488528 + nsecs: 690561056 +frame_id: "camera_color_optical_frame" +[INFO] [1734488529.155751]: received mask_obs with header seq: 1408 +stamp: + secs: 1734488529 + nsecs: 160276651 +frame_id: '' and grasped_object_mask header seq: 117897 +stamp: + secs: 1734488528 + nsecs: 890163422 +frame_id: "camera_color_optical_frame" +[INFO] [1734488529.355766]: received mask_obs with header seq: 1409 +stamp: + secs: 1734488529 + nsecs: 360211610 +frame_id: '' and grasped_object_mask header seq: 117904 +stamp: + secs: 1734488529 + nsecs: 123018265 +frame_id: "camera_color_optical_frame" +[INFO] [1734488529.553316]: received mask_obs with header seq: 1410 +stamp: + secs: 1734488529 + nsecs: 557853460 +frame_id: '' and grasped_object_mask header seq: 117910 +stamp: + secs: 1734488529 + nsecs: 322594881 +frame_id: "camera_color_optical_frame" +[INFO] [1734488529.749020]: received mask_obs with header seq: 1411 +stamp: + secs: 1734488529 + nsecs: 753644466 +frame_id: '' and grasped_object_mask header seq: 117915 +stamp: + secs: 1734488529 + nsecs: 488899469 +frame_id: "camera_color_optical_frame" +[INFO] [1734488529.938665]: received mask_obs with header seq: 1412 +stamp: + secs: 1734488529 + nsecs: 943445920 +frame_id: '' and grasped_object_mask header seq: 117921 +stamp: + secs: 1734488529 + nsecs: 688495874 +frame_id: "camera_color_optical_frame" +[INFO] [1734488530.138073]: received mask_obs with header seq: 1413 +stamp: + secs: 1734488530 + nsecs: 142867565 +frame_id: '' and grasped_object_mask header seq: 117927 +stamp: + secs: 1734488529 + nsecs: 888093948 +frame_id: "camera_color_optical_frame" +[INFO] [1734488530.337759]: received mask_obs with header seq: 1414 +stamp: + secs: 1734488530 + nsecs: 342407464 +frame_id: '' and grasped_object_mask header seq: 117933 +stamp: + secs: 1734488530 + nsecs: 87680101 +frame_id: "camera_color_optical_frame" +[INFO] [1734488530.537737]: received mask_obs with header seq: 1415 +stamp: + secs: 1734488530 + nsecs: 542540550 +frame_id: '' and grasped_object_mask header seq: 117939 +stamp: + secs: 1734488530 + nsecs: 287254333 +frame_id: "camera_color_optical_frame" +[INFO] [1734488530.737130]: received mask_obs with header seq: 1416 +stamp: + secs: 1734488530 + nsecs: 741718053 +frame_id: '' and grasped_object_mask header seq: 117945 +stamp: + secs: 1734488530 + nsecs: 486816645 +frame_id: "camera_color_optical_frame" +[INFO] [1734488530.937474]: received mask_obs with header seq: 1417 +stamp: + secs: 1734488530 + nsecs: 942238807 +frame_id: '' and grasped_object_mask header seq: 117951 +stamp: + secs: 1734488530 + nsecs: 687331676 +frame_id: "camera_color_optical_frame" +[INFO] [1734488531.137755]: received mask_obs with header seq: 1418 +stamp: + secs: 1734488531 + nsecs: 142568349 +frame_id: '' and grasped_object_mask header seq: 117957 +stamp: + secs: 1734488530 + nsecs: 888279915 +frame_id: "camera_color_optical_frame" +[INFO] [1734488531.341727]: received mask_obs with header seq: 1419 +stamp: + secs: 1734488531 + nsecs: 344391107 +frame_id: '' and grasped_object_mask header seq: 117963 +stamp: + secs: 1734488531 + nsecs: 89315891 +frame_id: "camera_color_optical_frame" +[INFO] [1734488531.537701]: received mask_obs with header seq: 1420 +stamp: + secs: 1734488531 + nsecs: 542112588 +frame_id: '' and grasped_object_mask header seq: 117969 +stamp: + secs: 1734488531 + nsecs: 290436983 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1734488531.601489]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1734488531.755733]: received mask_obs with header seq: 1421 +stamp: + secs: 1734488531 + nsecs: 760311365 +frame_id: '' and grasped_object_mask header seq: 117975 +stamp: + secs: 1734488531 + nsecs: 491644859 +frame_id: "camera_color_optical_frame" +[INFO] [1734488532.394552]: Stored episode 1. +[INFO] [1734488532.394810]: Saving video... +[INFO] [1734488532.586377]: Video saved! +Starting episode 1 +[INFO] [1734488532.600132]: resetting environment +[INFO] [1734488532.609962]: cleared current plan +[INFO] [1734488532.611135]: moving to home +[INFO] [1734488535.913657]: reached home +[INFO] [1734488535.914433]: reset action history +[INFO] [1734488538.624623]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734488557.040617]: resetting environment +[INFO] [1734488557.043843]: cleared current plan +[INFO] [1734488557.044026]: moving to home +[INFO] [1734488558.045287]: reached home +[INFO] [1734488558.046190]: reset action history +[INFO] [1734488560.753731]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_1_2024-12-17-21-22-40.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log +started roslaunch server http://158.130.50.37:44067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [76713] +process[print_text-5]: started with pid [76714] +process[rosbag_record-6]: started with pid [76715] +started bagging! +[INFO] [1734488561.427042]: received mask_obs with header seq: 1422 +stamp: + secs: 1734488561 + nsecs: 431577920 +frame_id: '' and grasped_object_mask header seq: 118867 +stamp: + secs: 1734488561 + nsecs: 201576233 +frame_id: "camera_color_optical_frame" +[INFO] [1734488561.622701]: received mask_obs with header seq: 1423 +stamp: + secs: 1734488561 + nsecs: 627487421 +frame_id: '' and grasped_object_mask header seq: 118873 +stamp: + secs: 1734488561 + nsecs: 401416063 +frame_id: "camera_color_optical_frame" +[INFO] [1734488561.821698]: received mask_obs with header seq: 1424 +stamp: + secs: 1734488561 + nsecs: 826492071 +frame_id: '' and grasped_object_mask header seq: 118879 +stamp: + secs: 1734488561 + nsecs: 601254463 +frame_id: "camera_color_optical_frame" +[INFO] [1734488562.032444]: received mask_obs with header seq: 1425 +stamp: + secs: 1734488562 + nsecs: 37048578 +frame_id: '' and grasped_object_mask header seq: 118885 +stamp: + secs: 1734488561 + nsecs: 801090956 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 118845038 for nearest: 1734488562119771771 - target: 1734488562000926733 at idx 15 +[INFO] [1734488562.249000]: received mask_obs with header seq: 1426 +stamp: + secs: 1734488562 + nsecs: 252330064 +frame_id: '' and grasped_object_mask header seq: 118891 +stamp: + secs: 1734488562 + nsecs: 926733 +frame_id: "camera_color_optical_frame" +[INFO] [1734488562.452958]: received mask_obs with header seq: 1427 +stamp: + secs: 1734488562 + nsecs: 457753896 +frame_id: '' and grasped_object_mask header seq: 118898 +stamp: + secs: 1734488562 + nsecs: 234067678 +frame_id: "camera_color_optical_frame" +[INFO] [1734488562.653386]: received mask_obs with header seq: 1428 +stamp: + secs: 1734488562 + nsecs: 657989740 +frame_id: '' and grasped_object_mask header seq: 118904 +stamp: + secs: 1734488562 + nsecs: 433904171 +frame_id: "camera_color_optical_frame" +[INFO] [1734488562.852416]: received mask_obs with header seq: 1429 +stamp: + secs: 1734488562 + nsecs: 856615066 +frame_id: '' and grasped_object_mask header seq: 118909 +stamp: + secs: 1734488562 + nsecs: 600434542 +frame_id: "camera_color_optical_frame" +[INFO] [1734488563.044461]: received mask_obs with header seq: 1430 +stamp: + secs: 1734488563 + nsecs: 49080848 +frame_id: '' and grasped_object_mask header seq: 118915 +stamp: + secs: 1734488562 + nsecs: 800274849 +frame_id: "camera_color_optical_frame" +[INFO] [1734488563.244241]: received mask_obs with header seq: 1431 +stamp: + secs: 1734488563 + nsecs: 249041557 +frame_id: '' and grasped_object_mask header seq: 118921 +stamp: + secs: 1734488563 + nsecs: 116348 +frame_id: "camera_color_optical_frame" +[INFO] [1734488563.443369]: received mask_obs with header seq: 1432 +stamp: + secs: 1734488563 + nsecs: 447961330 +frame_id: '' and grasped_object_mask header seq: 118927 +stamp: + secs: 1734488563 + nsecs: 199957132 +frame_id: "camera_color_optical_frame" +[INFO] [1734488563.643313]: received mask_obs with header seq: 1433 +stamp: + secs: 1734488563 + nsecs: 648080587 +frame_id: '' and grasped_object_mask header seq: 118933 +stamp: + secs: 1734488563 + nsecs: 399798155 +frame_id: "camera_color_optical_frame" +[INFO] [1734488563.842666]: received mask_obs with header seq: 1434 +stamp: + secs: 1734488563 + nsecs: 847209453 +frame_id: '' and grasped_object_mask header seq: 118939 +stamp: + secs: 1734488563 + nsecs: 599638939 +frame_id: "camera_color_optical_frame" +[INFO] [1734488564.063322]: received mask_obs with header seq: 1435 +stamp: + secs: 1734488564 + nsecs: 67942380 +frame_id: '' and grasped_object_mask header seq: 118945 +stamp: + secs: 1734488563 + nsecs: 799479485 +frame_id: "camera_color_optical_frame" +[INFO] [1734488564.266694]: received mask_obs with header seq: 1436 +stamp: + secs: 1734488564 + nsecs: 271458625 +frame_id: '' and grasped_object_mask header seq: 118952 +stamp: + secs: 1734488564 + nsecs: 32627344 +frame_id: "camera_color_optical_frame" +[INFO] [1734488564.464137]: received mask_obs with header seq: 1437 +stamp: + secs: 1734488564 + nsecs: 468806505 +frame_id: '' and grasped_object_mask header seq: 118958 +stamp: + secs: 1734488564 + nsecs: 232468367 +frame_id: "camera_color_optical_frame" +[INFO] [1734488564.662186]: received mask_obs with header seq: 1438 +stamp: + secs: 1734488564 + nsecs: 666749000 +frame_id: '' and grasped_object_mask header seq: 118964 +stamp: + secs: 1734488564 + nsecs: 432309151 +frame_id: "camera_color_optical_frame" +[INFO] [1734488564.856328]: received mask_obs with header seq: 1439 +stamp: + secs: 1734488564 + nsecs: 861112833 +frame_id: '' and grasped_object_mask header seq: 118970 +stamp: + secs: 1734488564 + nsecs: 632148027 +frame_id: "camera_color_optical_frame" +[INFO] [1734488565.057206]: received mask_obs with header seq: 1440 +stamp: + secs: 1734488565 + nsecs: 61779737 +frame_id: '' and grasped_object_mask header seq: 118976 +stamp: + secs: 1734488564 + nsecs: 831984520 +frame_id: "camera_color_optical_frame" +[INFO] [1734488565.251659]: received mask_obs with header seq: 1441 +stamp: + secs: 1734488565 + nsecs: 256386041 +frame_id: '' and grasped_object_mask header seq: 118981 +stamp: + secs: 1734488564 + nsecs: 998513699 +frame_id: "camera_color_optical_frame" +[INFO] [1734488565.446504]: received mask_obs with header seq: 1442 +stamp: + secs: 1734488565 + nsecs: 451287508 +frame_id: '' and grasped_object_mask header seq: 118987 +stamp: + secs: 1734488565 + nsecs: 198348761 +frame_id: "camera_color_optical_frame" +[INFO] [1734488565.643145]: received mask_obs with header seq: 1443 +stamp: + secs: 1734488565 + nsecs: 647970676 +frame_id: '' and grasped_object_mask header seq: 118993 +stamp: + secs: 1734488565 + nsecs: 398184061 +frame_id: "camera_color_optical_frame" +[INFO] [1734488565.846698]: received mask_obs with header seq: 1444 +stamp: + secs: 1734488565 + nsecs: 851200580 +frame_id: '' and grasped_object_mask header seq: 118999 +stamp: + secs: 1734488565 + nsecs: 598020077 +frame_id: "camera_color_optical_frame" +[INFO] [1734488566.047535]: received mask_obs with header seq: 1445 +stamp: + secs: 1734488566 + nsecs: 51820993 +frame_id: '' and grasped_object_mask header seq: 119005 +stamp: + secs: 1734488565 + nsecs: 797855377 +frame_id: "camera_color_optical_frame" +[INFO] [1734488566.243914]: received mask_obs with header seq: 1446 +stamp: + secs: 1734488566 + nsecs: 248727083 +frame_id: '' and grasped_object_mask header seq: 119011 +stamp: + secs: 1734488565 + nsecs: 997690916 +frame_id: "camera_color_optical_frame" +[INFO] [1734488566.444379]: received mask_obs with header seq: 1447 +stamp: + secs: 1734488566 + nsecs: 449194192 +frame_id: '' and grasped_object_mask header seq: 119017 +stamp: + secs: 1734488566 + nsecs: 197526217 +frame_id: "camera_color_optical_frame" +[INFO] [1734488566.649732]: received mask_obs with header seq: 1448 +stamp: + secs: 1734488566 + nsecs: 654295206 +frame_id: '' and grasped_object_mask header seq: 119023 +stamp: + secs: 1734488566 + nsecs: 397361279 +frame_id: "camera_color_optical_frame" +[INFO] [1734488566.848263]: received mask_obs with header seq: 1449 +stamp: + secs: 1734488566 + nsecs: 852911710 +frame_id: '' and grasped_object_mask header seq: 119029 +stamp: + secs: 1734488566 + nsecs: 597196817 +frame_id: "camera_color_optical_frame" +[INFO] [1734488567.047250]: received mask_obs with header seq: 1450 +stamp: + secs: 1734488567 + nsecs: 51189422 +frame_id: '' and grasped_object_mask header seq: 119035 +stamp: + secs: 1734488566 + nsecs: 797034979 +frame_id: "camera_color_optical_frame" +[INFO] [1734488567.246892]: received mask_obs with header seq: 1451 +stamp: + secs: 1734488567 + nsecs: 251092672 +frame_id: '' and grasped_object_mask header seq: 119041 +stamp: + secs: 1734488566 + nsecs: 996872187 +frame_id: "camera_color_optical_frame" +[INFO] [1734488567.453555]: received mask_obs with header seq: 1452 +stamp: + secs: 1734488567 + nsecs: 456952571 +frame_id: '' and grasped_object_mask header seq: 119047 +stamp: + secs: 1734488567 + nsecs: 196709156 +frame_id: "camera_color_optical_frame" +[INFO] [1734488567.656929]: received mask_obs with header seq: 1453 +stamp: + secs: 1734488567 + nsecs: 660795450 +frame_id: '' and grasped_object_mask header seq: 119053 +stamp: + secs: 1734488567 + nsecs: 396546364 +frame_id: "camera_color_optical_frame" +[INFO] [1734488567.856830]: received mask_obs with header seq: 1454 +stamp: + secs: 1734488567 + nsecs: 860838890 +frame_id: '' and grasped_object_mask header seq: 119060 +stamp: + secs: 1734488567 + nsecs: 629689455 +frame_id: "camera_color_optical_frame" +[INFO] [1734488568.060009]: received mask_obs with header seq: 1455 +stamp: + secs: 1734488568 + nsecs: 64021348 +frame_id: '' and grasped_object_mask header seq: 119065 +stamp: + secs: 1734488567 + nsecs: 796217680 +frame_id: "camera_color_optical_frame" +[INFO] [1734488568.260677]: received mask_obs with header seq: 1456 +stamp: + secs: 1734488568 + nsecs: 262897968 +frame_id: '' and grasped_object_mask header seq: 119072 +stamp: + secs: 1734488568 + nsecs: 29358387 +frame_id: "camera_color_optical_frame" +[INFO] [1734488568.457087]: received mask_obs with header seq: 1457 +stamp: + secs: 1734488568 + nsecs: 461876392 +frame_id: '' and grasped_object_mask header seq: 119078 +stamp: + secs: 1734488568 + nsecs: 229192734 +frame_id: "camera_color_optical_frame" +[INFO] [1734488568.653156]: received mask_obs with header seq: 1458 +stamp: + secs: 1734488568 + nsecs: 657963752 +frame_id: '' and grasped_object_mask header seq: 119083 +stamp: + secs: 1734488568 + nsecs: 395721674 +frame_id: "camera_color_optical_frame" +[INFO] [1734488568.846745]: received mask_obs with header seq: 1459 +stamp: + secs: 1734488568 + nsecs: 850787162 +frame_id: '' and grasped_object_mask header seq: 119089 +stamp: + secs: 1734488568 + nsecs: 595556736 +frame_id: "camera_color_optical_frame" +[INFO] [1734488569.046436]: received mask_obs with header seq: 1460 +stamp: + secs: 1734488569 + nsecs: 50691604 +frame_id: '' and grasped_object_mask header seq: 119095 +stamp: + secs: 1734488568 + nsecs: 795397043 +frame_id: "camera_color_optical_frame" +[INFO] [1734488569.243300]: received mask_obs with header seq: 1461 +stamp: + secs: 1734488569 + nsecs: 248096227 +frame_id: '' and grasped_object_mask header seq: 119101 +stamp: + secs: 1734488568 + nsecs: 995237589 +frame_id: "camera_color_optical_frame" +[INFO] [1734488569.452471]: received mask_obs with header seq: 1462 +stamp: + secs: 1734488569 + nsecs: 457210063 +frame_id: '' and grasped_object_mask header seq: 119107 +stamp: + secs: 1734488569 + nsecs: 195079088 +frame_id: "camera_color_optical_frame" +[INFO] [1734488569.660160]: received mask_obs with header seq: 1463 +stamp: + secs: 1734488569 + nsecs: 664606571 +frame_id: '' and grasped_object_mask header seq: 119113 +stamp: + secs: 1734488569 + nsecs: 394919634 +frame_id: "camera_color_optical_frame" +[INFO] [1734488569.859895]: received mask_obs with header seq: 1464 +stamp: + secs: 1734488569 + nsecs: 863925695 +frame_id: '' and grasped_object_mask header seq: 119120 +stamp: + secs: 1734488569 + nsecs: 628063917 +frame_id: "camera_color_optical_frame" +[INFO] [1734488570.058305]: received mask_obs with header seq: 1465 +stamp: + secs: 1734488570 + nsecs: 63014030 +frame_id: '' and grasped_object_mask header seq: 119126 +stamp: + secs: 1734488569 + nsecs: 827895403 +frame_id: "camera_color_optical_frame" +[INFO] [1734488570.255535]: received mask_obs with header seq: 1466 +stamp: + secs: 1734488570 + nsecs: 260270357 +frame_id: '' and grasped_object_mask header seq: 119132 +stamp: + secs: 1734488570 + nsecs: 27725935 +frame_id: "camera_color_optical_frame" +[INFO] [1734488570.448285]: received mask_obs with header seq: 1467 +stamp: + secs: 1734488570 + nsecs: 453035354 +frame_id: '' and grasped_object_mask header seq: 119137 +stamp: + secs: 1734488570 + nsecs: 194250584 +frame_id: "camera_color_optical_frame" +[INFO] [1734488570.649887]: received mask_obs with header seq: 1468 +stamp: + secs: 1734488570 + nsecs: 654668092 +frame_id: '' and grasped_object_mask header seq: 119143 +stamp: + secs: 1734488570 + nsecs: 394081116 +frame_id: "camera_color_optical_frame" +[INFO] [1734488570.851192]: received mask_obs with header seq: 1469 +stamp: + secs: 1734488570 + nsecs: 855838060 +frame_id: '' and grasped_object_mask header seq: 119149 +stamp: + secs: 1734488570 + nsecs: 593911171 +frame_id: "camera_color_optical_frame" +[INFO] [1734488571.051944]: received mask_obs with header seq: 1470 +stamp: + secs: 1734488571 + nsecs: 56265830 +frame_id: '' and grasped_object_mask header seq: 119155 +stamp: + secs: 1734488570 + nsecs: 793743849 +frame_id: "camera_color_optical_frame" +[INFO] [1734488571.250141]: received mask_obs with header seq: 1471 +stamp: + secs: 1734488571 + nsecs: 254881858 +frame_id: '' and grasped_object_mask header seq: 119161 +stamp: + secs: 1734488570 + nsecs: 993577003 +frame_id: "camera_color_optical_frame" +[INFO] [1734488571.449057]: received mask_obs with header seq: 1472 +stamp: + secs: 1734488571 + nsecs: 453782796 +frame_id: '' and grasped_object_mask header seq: 119167 +stamp: + secs: 1734488571 + nsecs: 193409443 +frame_id: "camera_color_optical_frame" +[INFO] [1734488571.649302]: received mask_obs with header seq: 1473 +stamp: + secs: 1734488571 + nsecs: 654060363 +frame_id: '' and grasped_object_mask header seq: 119174 +stamp: + secs: 1734488571 + nsecs: 426547527 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -106604662 for nearest: 1734488571519776974 - target: 1734488571626381636 at idx 146 +[INFO] [1734488571.846306]: received mask_obs with header seq: 1474 +stamp: + secs: 1734488571 + nsecs: 849535703 +frame_id: '' and grasped_object_mask header seq: 119179 +stamp: + secs: 1734488571 + nsecs: 593074799 +frame_id: "camera_color_optical_frame" +[INFO] [1734488572.050235]: received mask_obs with header seq: 1475 +stamp: + secs: 1734488572 + nsecs: 54821729 +frame_id: '' and grasped_object_mask header seq: 119185 +stamp: + secs: 1734488571 + nsecs: 792914391 +frame_id: "camera_color_optical_frame" +[INFO] [1734488572.250317]: received mask_obs with header seq: 1476 +stamp: + secs: 1734488572 + nsecs: 254938602 +frame_id: '' and grasped_object_mask header seq: 119191 +stamp: + secs: 1734488571 + nsecs: 992753029 +frame_id: "camera_color_optical_frame" +[INFO] [1734488572.444493]: received mask_obs with header seq: 1477 +stamp: + secs: 1734488572 + nsecs: 449152708 +frame_id: '' and grasped_object_mask header seq: 119197 +stamp: + secs: 1734488572 + nsecs: 192591906 +frame_id: "camera_color_optical_frame" +[INFO] [1734488572.651705]: received mask_obs with header seq: 1478 +stamp: + secs: 1734488572 + nsecs: 656347990 +frame_id: '' and grasped_object_mask header seq: 119204 +stamp: + secs: 1734488572 + nsecs: 425737619 +frame_id: "camera_color_optical_frame" +[INFO] [1734488572.846389]: received mask_obs with header seq: 1479 +stamp: + secs: 1734488572 + nsecs: 850614070 +frame_id: '' and grasped_object_mask header seq: 119209 +stamp: + secs: 1734488572 + nsecs: 592269421 +frame_id: "camera_color_optical_frame" +[INFO] [1734488573.059858]: received mask_obs with header seq: 1480 +stamp: + secs: 1734488573 + nsecs: 64097881 +frame_id: '' and grasped_object_mask header seq: 119216 +stamp: + secs: 1734488572 + nsecs: 825417042 +frame_id: "camera_color_optical_frame" +[INFO] [1734488573.263285]: received mask_obs with header seq: 1481 +stamp: + secs: 1734488573 + nsecs: 267984390 +frame_id: '' and grasped_object_mask header seq: 119222 +stamp: + secs: 1734488573 + nsecs: 25257587 +frame_id: "camera_color_optical_frame" +[INFO] [1734488573.460361]: received mask_obs with header seq: 1482 +stamp: + secs: 1734488573 + nsecs: 465042114 +frame_id: '' and grasped_object_mask header seq: 119228 +stamp: + secs: 1734488573 + nsecs: 225097895 +frame_id: "camera_color_optical_frame" +[INFO] [1734488573.670319]: received mask_obs with header seq: 1483 +stamp: + secs: 1734488573 + nsecs: 674854516 +frame_id: '' and grasped_object_mask header seq: 119234 +stamp: + secs: 1734488573 + nsecs: 424938440 +frame_id: "camera_color_optical_frame" +[INFO] [1734488573.859655]: received mask_obs with header seq: 1484 +stamp: + secs: 1734488573 + nsecs: 863938093 +frame_id: '' and grasped_object_mask header seq: 119240 +stamp: + secs: 1734488573 + nsecs: 624779940 +frame_id: "camera_color_optical_frame" +[INFO] [1734488574.060157]: received mask_obs with header seq: 1485 +stamp: + secs: 1734488574 + nsecs: 64419031 +frame_id: '' and grasped_object_mask header seq: 119246 +stamp: + secs: 1734488573 + nsecs: 824620962 +frame_id: "camera_color_optical_frame" +[INFO] [1734488574.255778]: received mask_obs with header seq: 1486 +stamp: + secs: 1734488574 + nsecs: 260196924 +frame_id: '' and grasped_object_mask header seq: 119252 +stamp: + secs: 1734488574 + nsecs: 24461746 +frame_id: "camera_color_optical_frame" +[INFO] [1734488574.449441]: received mask_obs with header seq: 1487 +stamp: + secs: 1734488574 + nsecs: 454118967 +frame_id: '' and grasped_object_mask header seq: 119257 +stamp: + secs: 1734488574 + nsecs: 190996647 +frame_id: "camera_color_optical_frame" +[INFO] [1734488574.646457]: received mask_obs with header seq: 1488 +stamp: + secs: 1734488574 + nsecs: 651059627 +frame_id: '' and grasped_object_mask header seq: 119264 +stamp: + secs: 1734488574 + nsecs: 424144030 +frame_id: "camera_color_optical_frame" +[INFO] [1734488574.846185]: received mask_obs with header seq: 1489 +stamp: + secs: 1734488574 + nsecs: 850825548 +frame_id: '' and grasped_object_mask header seq: 119270 +stamp: + secs: 1734488574 + nsecs: 623985529 +frame_id: "camera_color_optical_frame" +[INFO] [1734488575.050954]: received mask_obs with header seq: 1490 +stamp: + secs: 1734488575 + nsecs: 55735349 +frame_id: '' and grasped_object_mask header seq: 119276 +stamp: + secs: 1734488574 + nsecs: 823828459 +frame_id: "camera_color_optical_frame" +[INFO] [1734488575.245366]: received mask_obs with header seq: 1491 +stamp: + secs: 1734488575 + nsecs: 249246835 +frame_id: '' and grasped_object_mask header seq: 119282 +stamp: + secs: 1734488575 + nsecs: 23670435 +frame_id: "camera_color_optical_frame" +[INFO] [1734488575.446708]: received mask_obs with header seq: 1492 +stamp: + secs: 1734488575 + nsecs: 449606895 +frame_id: '' and grasped_object_mask header seq: 119288 +stamp: + secs: 1734488575 + nsecs: 223512650 +frame_id: "camera_color_optical_frame" +[INFO] [1734488575.645555]: received mask_obs with header seq: 1493 +stamp: + secs: 1734488575 + nsecs: 649819135 +frame_id: '' and grasped_object_mask header seq: 119293 +stamp: + secs: 1734488575 + nsecs: 390048504 +frame_id: "camera_color_optical_frame" +[INFO] [1734488575.860207]: received mask_obs with header seq: 1494 +stamp: + secs: 1734488575 + nsecs: 865018129 +frame_id: '' and grasped_object_mask header seq: 119300 +stamp: + secs: 1734488575 + nsecs: 623196363 +frame_id: "camera_color_optical_frame" +[INFO] [1734488576.058419]: received mask_obs with header seq: 1495 +stamp: + secs: 1734488576 + nsecs: 62748432 +frame_id: '' and grasped_object_mask header seq: 119306 +stamp: + secs: 1734488575 + nsecs: 823033810 +frame_id: "camera_color_optical_frame" +[INFO] [1734488576.254868]: received mask_obs with header seq: 1496 +stamp: + secs: 1734488576 + nsecs: 259683132 +frame_id: '' and grasped_object_mask header seq: 119312 +stamp: + secs: 1734488576 + nsecs: 22871733 +frame_id: "camera_color_optical_frame" +[INFO] [1734488576.450124]: received mask_obs with header seq: 1497 +stamp: + secs: 1734488576 + nsecs: 454505920 +frame_id: '' and grasped_object_mask header seq: 119318 +stamp: + secs: 1734488576 + nsecs: 222708941 +frame_id: "camera_color_optical_frame" +[INFO] [1734488576.650063]: received mask_obs with header seq: 1498 +stamp: + secs: 1734488576 + nsecs: 654848814 +frame_id: '' and grasped_object_mask header seq: 119324 +stamp: + secs: 1734488576 + nsecs: 422546148 +frame_id: "camera_color_optical_frame" +[INFO] [1734488576.850615]: received mask_obs with header seq: 1499 +stamp: + secs: 1734488576 + nsecs: 855276107 +frame_id: '' and grasped_object_mask header seq: 119330 +stamp: + secs: 1734488576 + nsecs: 622383356 +frame_id: "camera_color_optical_frame" +[INFO] [1734488577.053985]: received mask_obs with header seq: 1500 +stamp: + secs: 1734488577 + nsecs: 58812141 +frame_id: '' and grasped_object_mask header seq: 119336 +stamp: + secs: 1734488576 + nsecs: 822219610 +frame_id: "camera_color_optical_frame" +[INFO] [1734488577.246301]: received mask_obs with header seq: 1501 +stamp: + secs: 1734488577 + nsecs: 251011610 +frame_id: '' and grasped_object_mask header seq: 119342 +stamp: + secs: 1734488577 + nsecs: 22056341 +frame_id: "camera_color_optical_frame" +[INFO] [1734488577.451205]: received mask_obs with header seq: 1502 +stamp: + secs: 1734488577 + nsecs: 455959558 +frame_id: '' and grasped_object_mask header seq: 119348 +stamp: + secs: 1734488577 + nsecs: 221894264 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -101924911 for nearest: 1734488577319806084 - target: 1734488577421730995 at idx 225 +[INFO] [1734488577.650209]: received mask_obs with header seq: 1503 +stamp: + secs: 1734488577 + nsecs: 655004262 +frame_id: '' and grasped_object_mask header seq: 119354 +stamp: + secs: 1734488577 + nsecs: 421730995 +frame_id: "camera_color_optical_frame" +[INFO] [1734488577.851864]: received mask_obs with header seq: 1504 +stamp: + secs: 1734488577 + nsecs: 856585025 +frame_id: '' and grasped_object_mask header seq: 119360 +stamp: + secs: 1734488577 + nsecs: 621568918 +frame_id: "camera_color_optical_frame" +[INFO] [1734488578.053150]: received mask_obs with header seq: 1505 +stamp: + secs: 1734488578 + nsecs: 57719707 +frame_id: '' and grasped_object_mask header seq: 119365 +stamp: + secs: 1734488577 + nsecs: 788103819 +frame_id: "camera_color_optical_frame" +[INFO] [1734488578.249491]: received mask_obs with header seq: 1506 +stamp: + secs: 1734488578 + nsecs: 254211425 +frame_id: '' and grasped_object_mask header seq: 119372 +stamp: + secs: 1734488578 + nsecs: 21251440 +frame_id: "camera_color_optical_frame" +[INFO] [1734488578.460625]: received mask_obs with header seq: 1507 +stamp: + secs: 1734488578 + nsecs: 465170621 +frame_id: '' and grasped_object_mask header seq: 119378 +stamp: + secs: 1734488578 + nsecs: 221092701 +frame_id: "camera_color_optical_frame" +[INFO] [1734488578.661480]: received mask_obs with header seq: 1508 +stamp: + secs: 1734488578 + nsecs: 666278839 +frame_id: '' and grasped_object_mask header seq: 119384 +stamp: + secs: 1734488578 + nsecs: 420935392 +frame_id: "camera_color_optical_frame" +[INFO] [1734488578.869227]: received mask_obs with header seq: 1509 +stamp: + secs: 1734488578 + nsecs: 874022722 +frame_id: '' and grasped_object_mask header seq: 119390 +stamp: + secs: 1734488578 + nsecs: 620775223 +frame_id: "camera_color_optical_frame" +[INFO] [1734488579.072507]: received mask_obs with header seq: 1510 +stamp: + secs: 1734488579 + nsecs: 77324628 +frame_id: '' and grasped_object_mask header seq: 119396 +stamp: + secs: 1734488578 + nsecs: 820610046 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1734488579.137152]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1734488579.267859]: received mask_obs with header seq: 1511 +stamp: + secs: 1734488579 + nsecs: 272503852 +frame_id: '' and grasped_object_mask header seq: 119402 +stamp: + secs: 1734488579 + nsecs: 20444870 +frame_id: "camera_color_optical_frame" +[INFO] [1734488579.851763]: Stored episode 2. +[INFO] [1734488579.852542]: Saving video... +[INFO] [1734488580.370419]: Video saved! +Starting episode 2 +[INFO] [1734488580.413283]: resetting environment +[INFO] [1734488580.423526]: cleared current plan +[INFO] [1734488580.423770]: moving to home +[INFO] [1734488583.831673]: reached home +[INFO] [1734488583.834441]: reset action history +[INFO] [1734488586.602740]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734488600.975153]: resetting environment +[INFO] [1734488600.983774]: cleared current plan +[INFO] [1734488600.985259]: moving to home +[INFO] [1734488601.987035]: reached home +[INFO] [1734488601.987929]: reset action history +[INFO] [1734488604.695550]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/episode_rosbags/episode_2_2024-12-17-21-23-24.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-76272.log +started roslaunch server http://158.130.50.37:45511/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [76851] +process[print_text-8]: started with pid [76852] +process[rosbag_record-9]: started with pid [76853] +started bagging! +[INFO] [1734488605.362452]: received mask_obs with header seq: 1512 +stamp: + secs: 1734488605 + nsecs: 366941928 +frame_id: '' and grasped_object_mask header seq: 120187 +stamp: + secs: 1734488605 + nsecs: 174066544 +frame_id: "camera_color_optical_frame" +[INFO] [1734488605.571699]: received mask_obs with header seq: 1513 +stamp: + secs: 1734488605 + nsecs: 576406478 +frame_id: '' and grasped_object_mask header seq: 120193 +stamp: + secs: 1734488605 + nsecs: 373648882 +frame_id: "camera_color_optical_frame" +[INFO] [1734488605.768146]: received mask_obs with header seq: 1514 +stamp: + secs: 1734488605 + nsecs: 772837877 +frame_id: '' and grasped_object_mask header seq: 120199 +stamp: + secs: 1734488605 + nsecs: 573219299 +frame_id: "camera_color_optical_frame" +[INFO] [1734488605.974410]: received mask_obs with header seq: 1515 +stamp: + secs: 1734488605 + nsecs: 979129314 +frame_id: '' and grasped_object_mask header seq: 120205 +stamp: + secs: 1734488605 + nsecs: 772812843 +frame_id: "camera_color_optical_frame" +[INFO] [1734488606.177917]: received mask_obs with header seq: 1516 +stamp: + secs: 1734488606 + nsecs: 182721614 +frame_id: '' and grasped_object_mask header seq: 120211 +stamp: + secs: 1734488605 + nsecs: 972412109 +frame_id: "camera_color_optical_frame" +[INFO] [1734488606.383939]: received mask_obs with header seq: 1517 +stamp: + secs: 1734488606 + nsecs: 388672828 +frame_id: '' and grasped_object_mask header seq: 120217 +stamp: + secs: 1734488606 + nsecs: 172000408 +frame_id: "camera_color_optical_frame" +[INFO] [1734488606.587792]: received mask_obs with header seq: 1518 +stamp: + secs: 1734488606 + nsecs: 592413663 +frame_id: '' and grasped_object_mask header seq: 120223 +stamp: + secs: 1734488606 + nsecs: 371575832 +frame_id: "camera_color_optical_frame" +[INFO] [1734488606.791047]: received mask_obs with header seq: 1519 +stamp: + secs: 1734488606 + nsecs: 795864582 +frame_id: '' and grasped_object_mask header seq: 120229 +stamp: + secs: 1734488606 + nsecs: 571139574 +frame_id: "camera_color_optical_frame" +[INFO] [1734488606.992195]: received mask_obs with header seq: 1520 +stamp: + secs: 1734488606 + nsecs: 996987342 +frame_id: '' and grasped_object_mask header seq: 120235 +stamp: + secs: 1734488606 + nsecs: 770735264 +frame_id: "camera_color_optical_frame" +[INFO] [1734488607.197996]: received mask_obs with header seq: 1521 +stamp: + secs: 1734488607 + nsecs: 202747583 +frame_id: '' and grasped_object_mask header seq: 120241 +stamp: + secs: 1734488606 + nsecs: 970341206 +frame_id: "camera_color_optical_frame" +[INFO] [1734488607.394931]: received mask_obs with header seq: 1522 +stamp: + secs: 1734488607 + nsecs: 399050951 +frame_id: '' and grasped_object_mask header seq: 120247 +stamp: + secs: 1734488607 + nsecs: 169935703 +frame_id: "camera_color_optical_frame" +[INFO] [1734488607.595042]: received mask_obs with header seq: 1523 +stamp: + secs: 1734488607 + nsecs: 599728345 +frame_id: '' and grasped_object_mask header seq: 120253 +stamp: + secs: 1734488607 + nsecs: 369518280 +frame_id: "camera_color_optical_frame" +[INFO] [1734488607.792655]: received mask_obs with header seq: 1524 +stamp: + secs: 1734488607 + nsecs: 797485351 +frame_id: '' and grasped_object_mask header seq: 120259 +stamp: + secs: 1734488607 + nsecs: 569090366 +frame_id: "camera_color_optical_frame" +[INFO] [1734488607.994668]: received mask_obs with header seq: 1525 +stamp: + secs: 1734488607 + nsecs: 999407768 +frame_id: '' and grasped_object_mask header seq: 120265 +stamp: + secs: 1734488607 + nsecs: 768693924 +frame_id: "camera_color_optical_frame" +[INFO] [1734488608.193706]: received mask_obs with header seq: 1526 +stamp: + secs: 1734488608 + nsecs: 198462247 +frame_id: '' and grasped_object_mask header seq: 120271 +stamp: + secs: 1734488607 + nsecs: 968308449 +frame_id: "camera_color_optical_frame" +[INFO] [1734488608.392981]: received mask_obs with header seq: 1527 +stamp: + secs: 1734488608 + nsecs: 397598981 +frame_id: '' and grasped_object_mask header seq: 120277 +stamp: + secs: 1734488608 + nsecs: 167911530 +frame_id: "camera_color_optical_frame" +[INFO] [1734488608.593791]: received mask_obs with header seq: 1528 +stamp: + secs: 1734488608 + nsecs: 598575830 +frame_id: '' and grasped_object_mask header seq: 120283 +stamp: + secs: 1734488608 + nsecs: 367503405 +frame_id: "camera_color_optical_frame" +[INFO] [1734488608.793671]: received mask_obs with header seq: 1529 +stamp: + secs: 1734488608 + nsecs: 798375368 +frame_id: '' and grasped_object_mask header seq: 120289 +stamp: + secs: 1734488608 + nsecs: 567084312 +frame_id: "camera_color_optical_frame" +[INFO] [1734488608.993589]: received mask_obs with header seq: 1530 +stamp: + secs: 1734488608 + nsecs: 998181343 +frame_id: '' and grasped_object_mask header seq: 120295 +stamp: + secs: 1734488608 + nsecs: 766689301 +frame_id: "camera_color_optical_frame" +[INFO] [1734488609.193192]: received mask_obs with header seq: 1531 +stamp: + secs: 1734488609 + nsecs: 197862625 +frame_id: '' and grasped_object_mask header seq: 120301 +stamp: + secs: 1734488608 + nsecs: 966298819 +frame_id: "camera_color_optical_frame" +[INFO] [1734488609.392704]: received mask_obs with header seq: 1532 +stamp: + secs: 1734488609 + nsecs: 397487878 +frame_id: '' and grasped_object_mask header seq: 120307 +stamp: + secs: 1734488609 + nsecs: 165896893 +frame_id: "camera_color_optical_frame" +[INFO] [1734488609.592861]: received mask_obs with header seq: 1533 +stamp: + secs: 1734488609 + nsecs: 597646951 +frame_id: '' and grasped_object_mask header seq: 120313 +stamp: + secs: 1734488609 + nsecs: 365483761 +frame_id: "camera_color_optical_frame" +[INFO] [1734488609.792755]: received mask_obs with header seq: 1534 +stamp: + secs: 1734488609 + nsecs: 797553300 +frame_id: '' and grasped_object_mask header seq: 120319 +stamp: + secs: 1734488609 + nsecs: 565058947 +frame_id: "camera_color_optical_frame" +[INFO] [1734488609.995408]: received mask_obs with header seq: 1535 +stamp: + secs: 1734488610 + nsecs: 63419 +frame_id: '' and grasped_object_mask header seq: 120325 +stamp: + secs: 1734488609 + nsecs: 764657974 +frame_id: "camera_color_optical_frame" +[INFO] [1734488610.192667]: received mask_obs with header seq: 1536 +stamp: + secs: 1734488610 + nsecs: 197435617 +frame_id: '' and grasped_object_mask header seq: 120331 +stamp: + secs: 1734488609 + nsecs: 964264393 +frame_id: "camera_color_optical_frame" +[INFO] [1734488610.393017]: received mask_obs with header seq: 1537 +stamp: + secs: 1734488610 + nsecs: 397775173 +frame_id: '' and grasped_object_mask header seq: 120337 +stamp: + secs: 1734488610 + nsecs: 163858414 +frame_id: "camera_color_optical_frame" +[INFO] [1734488610.603862]: received mask_obs with header seq: 1538 +stamp: + secs: 1734488610 + nsecs: 608621120 +frame_id: '' and grasped_object_mask header seq: 120343 +stamp: + secs: 1734488610 + nsecs: 363440514 +frame_id: "camera_color_optical_frame" +[INFO] [1734488610.814958]: received mask_obs with header seq: 1539 +stamp: + secs: 1734488610 + nsecs: 819467306 +frame_id: '' and grasped_object_mask header seq: 120350 +stamp: + secs: 1734488610 + nsecs: 596272230 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.004504]: received mask_obs with header seq: 1540 +stamp: + secs: 1734488611 + nsecs: 8985519 +frame_id: '' and grasped_object_mask header seq: 120355 +stamp: + secs: 1734488610 + nsecs: 762609005 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.199563]: received mask_obs with header seq: 1541 +stamp: + secs: 1734488611 + nsecs: 204004526 +frame_id: '' and grasped_object_mask header seq: 120361 +stamp: + secs: 1734488610 + nsecs: 962214708 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.391644]: received mask_obs with header seq: 1542 +stamp: + secs: 1734488611 + nsecs: 396319389 +frame_id: '' and grasped_object_mask header seq: 120367 +stamp: + secs: 1734488611 + nsecs: 161809683 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.591723]: received mask_obs with header seq: 1543 +stamp: + secs: 1734488611 + nsecs: 596360206 +frame_id: '' and grasped_object_mask header seq: 120373 +stamp: + secs: 1734488611 + nsecs: 361391783 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.791330]: received mask_obs with header seq: 1544 +stamp: + secs: 1734488611 + nsecs: 795974969 +frame_id: '' and grasped_object_mask header seq: 120379 +stamp: + secs: 1734488611 + nsecs: 560962439 +frame_id: "camera_color_optical_frame" +[INFO] [1734488611.992624]: received mask_obs with header seq: 1545 +stamp: + secs: 1734488611 + nsecs: 997135162 +frame_id: '' and grasped_object_mask header seq: 120385 +stamp: + secs: 1734488611 + nsecs: 760560989 +frame_id: "camera_color_optical_frame" +[INFO] [1734488612.193075]: received mask_obs with header seq: 1546 +stamp: + secs: 1734488612 + nsecs: 197808742 +frame_id: '' and grasped_object_mask header seq: 120391 +stamp: + secs: 1734488611 + nsecs: 960167646 +frame_id: "camera_color_optical_frame" +[INFO] [1734488612.392593]: received mask_obs with header seq: 1547 +stamp: + secs: 1734488612 + nsecs: 397080659 +frame_id: '' and grasped_object_mask header seq: 120397 +stamp: + secs: 1734488612 + nsecs: 159762621 +frame_id: "camera_color_optical_frame" +[INFO] [1734488612.591845]: received mask_obs with header seq: 1548 +stamp: + secs: 1734488612 + nsecs: 596584081 +frame_id: '' and grasped_object_mask header seq: 120403 +stamp: + secs: 1734488612 + nsecs: 359346867 +frame_id: "camera_color_optical_frame" +[INFO] [1734488612.791497]: received mask_obs with header seq: 1549 +stamp: + secs: 1734488612 + nsecs: 795956850 +frame_id: '' and grasped_object_mask header seq: 120409 +stamp: + secs: 1734488612 + nsecs: 558918476 +frame_id: "camera_color_optical_frame" +[INFO] [1734488612.991452]: received mask_obs with header seq: 1550 +stamp: + secs: 1734488612 + nsecs: 995924949 +frame_id: '' and grasped_object_mask header seq: 120415 +stamp: + secs: 1734488612 + nsecs: 758515596 +frame_id: "camera_color_optical_frame" +[INFO] [1734488613.191323]: received mask_obs with header seq: 1551 +stamp: + secs: 1734488613 + nsecs: 195973157 +frame_id: '' and grasped_object_mask header seq: 120421 +stamp: + secs: 1734488612 + nsecs: 958119154 +frame_id: "camera_color_optical_frame" +[INFO] [1734488613.392769]: received mask_obs with header seq: 1552 +stamp: + secs: 1734488613 + nsecs: 397516250 +frame_id: '' and grasped_object_mask header seq: 120427 +stamp: + secs: 1734488613 + nsecs: 157711029 +frame_id: "camera_color_optical_frame" +[INFO] [1734488613.600884]: received mask_obs with header seq: 1553 +stamp: + secs: 1734488613 + nsecs: 605432987 +frame_id: '' and grasped_object_mask header seq: 120433 +stamp: + secs: 1734488613 + nsecs: 357290745 +frame_id: "camera_color_optical_frame" +[INFO] [1734488613.816622]: received mask_obs with header seq: 1554 +stamp: + secs: 1734488613 + nsecs: 821230411 +frame_id: '' and grasped_object_mask header seq: 120439 +stamp: + secs: 1734488613 + nsecs: 556859970 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.011085]: received mask_obs with header seq: 1555 +stamp: + secs: 1734488614 + nsecs: 15702962 +frame_id: '' and grasped_object_mask header seq: 120445 +stamp: + secs: 1734488613 + nsecs: 756456137 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.200415]: received mask_obs with header seq: 1556 +stamp: + secs: 1734488614 + nsecs: 205089569 +frame_id: '' and grasped_object_mask header seq: 120451 +stamp: + secs: 1734488613 + nsecs: 956060171 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.394276]: received mask_obs with header seq: 1557 +stamp: + secs: 1734488614 + nsecs: 398661136 +frame_id: '' and grasped_object_mask header seq: 120457 +stamp: + secs: 1734488614 + nsecs: 155652285 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.592327]: received mask_obs with header seq: 1558 +stamp: + secs: 1734488614 + nsecs: 597019910 +frame_id: '' and grasped_object_mask header seq: 120463 +stamp: + secs: 1734488614 + nsecs: 355232954 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.792209]: received mask_obs with header seq: 1559 +stamp: + secs: 1734488614 + nsecs: 796582460 +frame_id: '' and grasped_object_mask header seq: 120469 +stamp: + secs: 1734488614 + nsecs: 554801464 +frame_id: "camera_color_optical_frame" +[INFO] [1734488614.993536]: received mask_obs with header seq: 1560 +stamp: + secs: 1734488614 + nsecs: 998095512 +frame_id: '' and grasped_object_mask header seq: 120475 +stamp: + secs: 1734488614 + nsecs: 754396677 +frame_id: "camera_color_optical_frame" +[INFO] [1734488615.193352]: received mask_obs with header seq: 1561 +stamp: + secs: 1734488615 + nsecs: 198084354 +frame_id: '' and grasped_object_mask header seq: 120481 +stamp: + secs: 1734488614 + nsecs: 953999996 +frame_id: "camera_color_optical_frame" +[INFO] [1734488615.393521]: received mask_obs with header seq: 1562 +stamp: + secs: 1734488615 + nsecs: 398257493 +frame_id: '' and grasped_object_mask header seq: 120487 +stamp: + secs: 1734488615 + nsecs: 153590441 +frame_id: "camera_color_optical_frame" +[INFO] [1734488615.591379]: received mask_obs with header seq: 1563 +stamp: + secs: 1734488615 + nsecs: 596136331 +frame_id: '' and grasped_object_mask header seq: 120493 +stamp: + secs: 1734488615 + nsecs: 353168964 +frame_id: "camera_color_optical_frame" +[INFO] [1734488615.793644]: received mask_obs with header seq: 1564 +stamp: + secs: 1734488615 + nsecs: 798228263 +frame_id: '' and grasped_object_mask header seq: 120499 +stamp: + secs: 1734488615 + nsecs: 552735567 +frame_id: "camera_color_optical_frame" +[INFO] [1734488615.993771]: received mask_obs with header seq: 1565 +stamp: + secs: 1734488615 + nsecs: 998428106 +frame_id: '' and grasped_object_mask header seq: 120505 +stamp: + secs: 1734488615 + nsecs: 752336502 +frame_id: "camera_color_optical_frame" +[INFO] [1734488616.193889]: received mask_obs with header seq: 1566 +stamp: + secs: 1734488616 + nsecs: 198642253 +frame_id: '' and grasped_object_mask header seq: 120511 +stamp: + secs: 1734488615 + nsecs: 951948643 +frame_id: "camera_color_optical_frame" +[INFO] [1734488616.393542]: received mask_obs with header seq: 1567 +stamp: + secs: 1734488616 + nsecs: 398267984 +frame_id: '' and grasped_object_mask header seq: 120517 +stamp: + secs: 1734488616 + nsecs: 151550531 +frame_id: "camera_color_optical_frame" +[INFO] [1734488616.593372]: received mask_obs with header seq: 1568 +stamp: + secs: 1734488616 + nsecs: 597777128 +frame_id: '' and grasped_object_mask header seq: 120523 +stamp: + secs: 1734488616 + nsecs: 351139784 +frame_id: "camera_color_optical_frame" +[INFO] [1734488616.806933]: received mask_obs with header seq: 1569 +stamp: + secs: 1734488616 + nsecs: 811230421 +frame_id: '' and grasped_object_mask header seq: 120529 +stamp: + secs: 1734488616 + nsecs: 550718069 +frame_id: "camera_color_optical_frame" +[INFO] [1734488617.015355]: received mask_obs with header seq: 1570 +stamp: + secs: 1734488617 + nsecs: 19805908 +frame_id: '' and grasped_object_mask header seq: 120535 +stamp: + secs: 1734488616 + nsecs: 751157999 +frame_id: "camera_color_optical_frame" +[INFO] [1734488617.216903]: received mask_obs with header seq: 1571 +stamp: + secs: 1734488617 + nsecs: 221486568 +frame_id: '' and grasped_object_mask header seq: 120541 +stamp: + secs: 1734488616 + nsecs: 952105284 +frame_id: "camera_color_optical_frame" +[INFO] [1734488617.420572]: received mask_obs with header seq: 1572 +stamp: + secs: 1734488617 + nsecs: 425257205 +frame_id: '' and grasped_object_mask header seq: 120548 +stamp: + secs: 1734488617 + nsecs: 186654091 +frame_id: "camera_color_optical_frame" +[INFO] [1734488617.620439]: received mask_obs with header seq: 1573 +stamp: + secs: 1734488617 + nsecs: 624833583 +frame_id: '' and grasped_object_mask header seq: 120554 +stamp: + secs: 1734488617 + nsecs: 387789488 +frame_id: "camera_color_optical_frame" +[INFO] [1734488617.815166]: received mask_obs with header seq: 1574 +stamp: + secs: 1734488617 + nsecs: 819962024 +frame_id: '' and grasped_object_mask header seq: 120560 +stamp: + secs: 1734488617 + nsecs: 589011669 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.014327]: received mask_obs with header seq: 1575 +stamp: + secs: 1734488618 + nsecs: 18929243 +frame_id: '' and grasped_object_mask header seq: 120566 +stamp: + secs: 1734488617 + nsecs: 789096355 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.212758]: received mask_obs with header seq: 1576 +stamp: + secs: 1734488618 + nsecs: 217058658 +frame_id: '' and grasped_object_mask header seq: 120572 +stamp: + secs: 1734488617 + nsecs: 988945007 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.404349]: received mask_obs with header seq: 1577 +stamp: + secs: 1734488618 + nsecs: 408903121 +frame_id: '' and grasped_object_mask header seq: 120577 +stamp: + secs: 1734488618 + nsecs: 155484676 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.594722]: received mask_obs with header seq: 1578 +stamp: + secs: 1734488618 + nsecs: 599435806 +frame_id: '' and grasped_object_mask header seq: 120583 +stamp: + secs: 1734488618 + nsecs: 355333567 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.794130]: received mask_obs with header seq: 1579 +stamp: + secs: 1734488618 + nsecs: 798756361 +frame_id: '' and grasped_object_mask header seq: 120589 +stamp: + secs: 1734488618 + nsecs: 555183887 +frame_id: "camera_color_optical_frame" +[INFO] [1734488618.994119]: received mask_obs with header seq: 1580 +stamp: + secs: 1734488618 + nsecs: 998494863 +frame_id: '' and grasped_object_mask header seq: 120595 +stamp: + secs: 1734488618 + nsecs: 755027771 +frame_id: "camera_color_optical_frame" +[INFO] [1734488619.193269]: received mask_obs with header seq: 1581 +stamp: + secs: 1734488619 + nsecs: 198017835 +frame_id: '' and grasped_object_mask header seq: 120601 +stamp: + secs: 1734488618 + nsecs: 954868793 +frame_id: "camera_color_optical_frame" +[INFO] [1734488619.394068]: received mask_obs with header seq: 1582 +stamp: + secs: 1734488619 + nsecs: 398792028 +frame_id: '' and grasped_object_mask header seq: 120607 +stamp: + secs: 1734488619 + nsecs: 154710293 +frame_id: "camera_color_optical_frame" +[INFO] [1734488619.594548]: received mask_obs with header seq: 1583 +stamp: + secs: 1734488619 + nsecs: 599262952 +frame_id: '' and grasped_object_mask header seq: 120613 +stamp: + secs: 1734488619 + nsecs: 354551554 +frame_id: "camera_color_optical_frame" +[INFO] [1734488619.806545]: received mask_obs with header seq: 1584 +stamp: + secs: 1734488619 + nsecs: 810845851 +frame_id: '' and grasped_object_mask header seq: 120619 +stamp: + secs: 1734488619 + nsecs: 554392815 +frame_id: "camera_color_optical_frame" +[INFO] [1734488619.998449]: received mask_obs with header seq: 1585 +stamp: + secs: 1734488620 + nsecs: 2909898 +frame_id: '' and grasped_object_mask header seq: 120625 +stamp: + secs: 1734488619 + nsecs: 754235744 +frame_id: "camera_color_optical_frame" +[INFO] [1734488620.194699]: received mask_obs with header seq: 1586 +stamp: + secs: 1734488620 + nsecs: 199217081 +frame_id: '' and grasped_object_mask header seq: 120631 +stamp: + secs: 1734488619 + nsecs: 954078197 +frame_id: "camera_color_optical_frame" +[INFO] [1734488620.395786]: received mask_obs with header seq: 1587 +stamp: + secs: 1734488620 + nsecs: 400550842 +frame_id: '' and grasped_object_mask header seq: 120637 +stamp: + secs: 1734488620 + nsecs: 153920412 +frame_id: "camera_color_optical_frame" +[INFO] [1734488620.594914]: received mask_obs with header seq: 1588 +stamp: + secs: 1734488620 + nsecs: 599648237 +frame_id: '' and grasped_object_mask header seq: 120643 +stamp: + secs: 1734488620 + nsecs: 353762865 +frame_id: "camera_color_optical_frame" +[INFO] [1734488620.795459]: received mask_obs with header seq: 1589 +stamp: + secs: 1734488620 + nsecs: 800186157 +frame_id: '' and grasped_object_mask header seq: 120649 +stamp: + secs: 1734488620 + nsecs: 553605556 +frame_id: "camera_color_optical_frame" +[INFO] [1734488620.995793]: received mask_obs with header seq: 1590 +stamp: + secs: 1734488621 + nsecs: 473260 +frame_id: '' and grasped_object_mask header seq: 120655 +stamp: + secs: 1734488620 + nsecs: 753448009 +frame_id: "camera_color_optical_frame" +[INFO] [1734488621.196666]: received mask_obs with header seq: 1591 +stamp: + secs: 1734488621 + nsecs: 201425552 +frame_id: '' and grasped_object_mask header seq: 120661 +stamp: + secs: 1734488620 + nsecs: 953291178 +frame_id: "camera_color_optical_frame" +[INFO] [1734488621.393859]: received mask_obs with header seq: 1592 +stamp: + secs: 1734488621 + nsecs: 398572206 +frame_id: '' and grasped_object_mask header seq: 120667 +stamp: + secs: 1734488621 + nsecs: 153134108 +frame_id: "camera_color_optical_frame" +[INFO] [1734488621.595337]: received mask_obs with header seq: 1593 +stamp: + secs: 1734488621 + nsecs: 599834918 +frame_id: '' and grasped_object_mask header seq: 120673 +stamp: + secs: 1734488621 + nsecs: 352976799 +frame_id: "camera_color_optical_frame" +[INFO] [1734488621.796055]: received mask_obs with header seq: 1594 +stamp: + secs: 1734488621 + nsecs: 800581693 +frame_id: '' and grasped_object_mask header seq: 120679 +stamp: + secs: 1734488621 + nsecs: 552819729 +frame_id: "camera_color_optical_frame" +[INFO] [1734488621.997539]: received mask_obs with header seq: 1595 +stamp: + secs: 1734488622 + nsecs: 1941680 +frame_id: '' and grasped_object_mask header seq: 120685 +stamp: + secs: 1734488621 + nsecs: 752654314 +frame_id: "camera_color_optical_frame" +[INFO] [1734488622.213064]: received mask_obs with header seq: 1596 +stamp: + secs: 1734488622 + nsecs: 217785358 +frame_id: '' and grasped_object_mask header seq: 120691 +stamp: + secs: 1734488621 + nsecs: 952483654 +frame_id: "camera_color_optical_frame" +[INFO] [1734488622.412414]: received mask_obs with header seq: 1597 +stamp: + secs: 1734488622 + nsecs: 417165279 +frame_id: '' and grasped_object_mask header seq: 120698 +stamp: + secs: 1734488622 + nsecs: 185618162 +frame_id: "camera_color_optical_frame" +[INFO] [1734488622.597838]: received mask_obs with header seq: 1598 +stamp: + secs: 1734488622 + nsecs: 602610349 +frame_id: '' and grasped_object_mask header seq: 120703 +stamp: + secs: 1734488622 + nsecs: 352142334 +frame_id: "camera_color_optical_frame" +[INFO] [1734488622.796506]: received mask_obs with header seq: 1599 +stamp: + secs: 1734488622 + nsecs: 801192522 +frame_id: '' and grasped_object_mask header seq: 120709 +stamp: + secs: 1734488622 + nsecs: 551970482 +frame_id: "camera_color_optical_frame" +[INFO] [1734488623.015378]: received mask_obs with header seq: 1600 +stamp: + secs: 1734488623 + nsecs: 20085573 +frame_id: '' and grasped_object_mask header seq: 120715 +stamp: + secs: 1734488622 + nsecs: 751805067 +frame_id: "camera_color_optical_frame" +[INFO] [1734488623.213866]: received mask_obs with header seq: 1601 +stamp: + secs: 1734488623 + nsecs: 218625068 +frame_id: '' and grasped_object_mask header seq: 120722 +stamp: + secs: 1734488622 + nsecs: 984950304 +frame_id: "camera_color_optical_frame" +[INFO] [1734488623.408698]: received mask_obs with header seq: 1602 +stamp: + secs: 1734488623 + nsecs: 413304090 +frame_id: '' and grasped_object_mask header seq: 120728 +stamp: + secs: 1734488623 + nsecs: 184788227 +frame_id: "camera_color_optical_frame" +[INFO] [1734488623.600203]: received mask_obs with header seq: 1603 +stamp: + secs: 1734488623 + nsecs: 604870796 +frame_id: '' and grasped_object_mask header seq: 120734 +stamp: + secs: 1734488623 + nsecs: 384626389 +frame_id: "camera_color_optical_frame" +[INFO] [1734488623.813736]: received mask_obs with header seq: 1604 +stamp: + secs: 1734488623 + nsecs: 818533658 +frame_id: '' and grasped_object_mask header seq: 120739 +stamp: + secs: 1734488623 + nsecs: 551158905 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.014790]: received mask_obs with header seq: 1605 +stamp: + secs: 1734488624 + nsecs: 19541263 +frame_id: '' and grasped_object_mask header seq: 120746 +stamp: + secs: 1734488623 + nsecs: 784304142 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.212339]: received mask_obs with header seq: 1606 +stamp: + secs: 1734488624 + nsecs: 217118501 +frame_id: '' and grasped_object_mask header seq: 120752 +stamp: + secs: 1734488623 + nsecs: 984144211 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.410587]: received mask_obs with header seq: 1607 +stamp: + secs: 1734488624 + nsecs: 415372610 +frame_id: '' and grasped_object_mask header seq: 120758 +stamp: + secs: 1734488624 + nsecs: 183985233 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.609923]: received mask_obs with header seq: 1608 +stamp: + secs: 1734488624 + nsecs: 614696025 +frame_id: '' and grasped_object_mask header seq: 120764 +stamp: + secs: 1734488624 + nsecs: 383825779 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.802581]: received mask_obs with header seq: 1609 +stamp: + secs: 1734488624 + nsecs: 807047843 +frame_id: '' and grasped_object_mask header seq: 120770 +stamp: + secs: 1734488624 + nsecs: 583666325 +frame_id: "camera_color_optical_frame" +[INFO] [1734488624.994152]: received mask_obs with header seq: 1610 +stamp: + secs: 1734488624 + nsecs: 998709440 +frame_id: '' and grasped_object_mask header seq: 120775 +stamp: + secs: 1734488624 + nsecs: 750202179 +frame_id: "camera_color_optical_frame" +[INFO] [1734488625.194065]: received mask_obs with header seq: 1611 +stamp: + secs: 1734488625 + nsecs: 198657512 +frame_id: '' and grasped_object_mask header seq: 120781 +stamp: + secs: 1734488624 + nsecs: 950044870 +frame_id: "camera_color_optical_frame" +[INFO] [1734488625.394781]: received mask_obs with header seq: 1612 +stamp: + secs: 1734488625 + nsecs: 399248838 +frame_id: '' and grasped_object_mask header seq: 120787 +stamp: + secs: 1734488625 + nsecs: 149888277 +frame_id: "camera_color_optical_frame" +[INFO] [1734488625.594160]: received mask_obs with header seq: 1613 +stamp: + secs: 1734488625 + nsecs: 598845720 +frame_id: '' and grasped_object_mask header seq: 120793 +stamp: + secs: 1734488625 + nsecs: 349731684 +frame_id: "camera_color_optical_frame" +[INFO] [1734488625.811262]: received mask_obs with header seq: 1614 +stamp: + secs: 1734488625 + nsecs: 816011667 +frame_id: '' and grasped_object_mask header seq: 120799 +stamp: + secs: 1734488625 + nsecs: 549575090 +frame_id: "camera_color_optical_frame" +[INFO] [1734488626.012267]: received mask_obs with header seq: 1615 +stamp: + secs: 1734488626 + nsecs: 16935825 +frame_id: '' and grasped_object_mask header seq: 120806 +stamp: + secs: 1734488625 + nsecs: 782722473 +frame_id: "camera_color_optical_frame" +[INFO] [1734488626.205694]: received mask_obs with header seq: 1616 +stamp: + secs: 1734488626 + nsecs: 210468769 +frame_id: '' and grasped_object_mask header seq: 120812 +stamp: + secs: 1734488625 + nsecs: 982560635 +frame_id: "camera_color_optical_frame" +[INFO] [1734488626.405659]: received mask_obs with header seq: 1617 +stamp: + secs: 1734488626 + nsecs: 409345626 +frame_id: '' and grasped_object_mask header seq: 120817 +stamp: + secs: 1734488626 + nsecs: 149092913 +frame_id: "camera_color_optical_frame" +[INFO] [1734488626.620202]: received mask_obs with header seq: 1618 +stamp: + secs: 1734488626 + nsecs: 624992847 +frame_id: '' and grasped_object_mask header seq: 120824 +stamp: + secs: 1734488626 + nsecs: 382236958 +frame_id: "camera_color_optical_frame" +[INFO] [1734488626.815510]: received mask_obs with header seq: 1619 +stamp: + secs: 1734488626 + nsecs: 820239305 +frame_id: '' and grasped_object_mask header seq: 120830 +stamp: + secs: 1734488626 + nsecs: 582076073 +frame_id: "camera_color_optical_frame" +[INFO] [1734488627.011742]: received mask_obs with header seq: 1620 +stamp: + secs: 1734488627 + nsecs: 16483783 +frame_id: '' and grasped_object_mask header seq: 120836 +stamp: + secs: 1734488626 + nsecs: 781913757 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -261975177 for nearest: 1734488626719776027 - target: 1734488626981751204 at idx 249 +[INFO] [1734488627.210031]: received mask_obs with header seq: 1621 +stamp: + secs: 1734488627 + nsecs: 214428663 +frame_id: '' and grasped_object_mask header seq: 120842 +stamp: + secs: 1734488626 + nsecs: 981751204 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -461812384 for nearest: 1734488626719776027 - target: 1734488627181588411 at idx 249 +[INFO] [1734488627.408113]: received mask_obs with header seq: 1622 +stamp: + secs: 1734488627 + nsecs: 412234306 +frame_id: '' and grasped_object_mask header seq: 120848 +stamp: + secs: 1734488627 + nsecs: 181588411 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -661649831 for nearest: 1734488626719776027 - target: 1734488627381425858 at idx 249 +[INFO] [1734488627.602649]: received mask_obs with header seq: 1623 +stamp: + secs: 1734488627 + nsecs: 606830120 +frame_id: '' and grasped_object_mask header seq: 120854 +stamp: + secs: 1734488627 + nsecs: 381425858 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -861486562 for nearest: 1734488626719776027 - target: 1734488627581262589 at idx 249 +[INFO] [1734488627.797651]: received mask_obs with header seq: 1624 +stamp: + secs: 1734488627 + nsecs: 802087783 +frame_id: '' and grasped_object_mask header seq: 120859 +stamp: + secs: 1734488627 + nsecs: 547956944 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -1061324961 for nearest: 1734488626719776027 - target: 1734488627781100988 at idx 249 +[INFO] [1734488628.002376]: received mask_obs with header seq: 1625 +stamp: + secs: 1734488628 + nsecs: 7000923 +frame_id: '' and grasped_object_mask header seq: 120865 +stamp: + secs: 1734488627 + nsecs: 747794390 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -1261161931 for nearest: 1734488626719776027 - target: 1734488627980937958 at idx 249 +[INFO] [1734488628.197935]: received mask_obs with header seq: 1626 +stamp: + secs: 1734488628 + nsecs: 202474594 +frame_id: '' and grasped_object_mask header seq: 120871 +stamp: + secs: 1734488627 + nsecs: 947631359 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -1460998900 for nearest: 1734488626719776027 - target: 1734488628180774927 at idx 249 +[INFO] [1734488628.397832]: received mask_obs with header seq: 1627 +stamp: + secs: 1734488628 + nsecs: 401948928 +frame_id: '' and grasped_object_mask header seq: 120877 +stamp: + secs: 1734488628 + nsecs: 147468328 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -1660835870 for nearest: 1734488626719776027 - target: 1734488628380611897 at idx 249 +[INFO] [1734488628.597165]: received mask_obs with header seq: 1628 +stamp: + secs: 1734488628 + nsecs: 601340055 +frame_id: '' and grasped_object_mask header seq: 120883 +stamp: + secs: 1734488628 + nsecs: 347305059 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -1860672600 for nearest: 1734488626719776027 - target: 1734488628580448627 at idx 249 +[INFO] [1734488628.801036]: received mask_obs with header seq: 1629 +stamp: + secs: 1734488628 + nsecs: 805765151 +frame_id: '' and grasped_object_mask header seq: 120890 +stamp: + secs: 1734488628 + nsecs: 580448627 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -2027202733 for nearest: 1734488626719776027 - target: 1734488628746978760 at idx 249 +[INFO] [1734488629.006799]: received mask_obs with header seq: 1630 +stamp: + secs: 1734488629 + nsecs: 11545181 +frame_id: '' and grasped_object_mask header seq: 120895 +stamp: + secs: 1734488628 + nsecs: 746978760 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -2260343440 for nearest: 1734488626719776027 - target: 1734488628980119467 at idx 249 +[INFO] [1734488629.210676]: received mask_obs with header seq: 1631 +stamp: + secs: 1734488629 + nsecs: 215385913 +frame_id: '' and grasped_object_mask header seq: 120902 +stamp: + secs: 1734488628 + nsecs: 980119467 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -2460177548 for nearest: 1734488626719776027 - target: 1734488629179953575 at idx 249 +[INFO] [1734488629.407640]: received mask_obs with header seq: 1632 +stamp: + secs: 1734488629 + nsecs: 412182569 +frame_id: '' and grasped_object_mask header seq: 120908 +stamp: + secs: 1734488629 + nsecs: 179953575 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -2660011656 for nearest: 1734488626719776027 - target: 1734488629379787683 at idx 249 +[INFO] [1734488629.607066]: received mask_obs with header seq: 1633 +stamp: + secs: 1734488629 + nsecs: 611466646 +frame_id: '' and grasped_object_mask header seq: 120914 +stamp: + secs: 1734488629 + nsecs: 379787683 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -2859845288 for nearest: 1734488626719776027 - target: 1734488629579621315 at idx 249 +[INFO] [1734488629.805089]: received mask_obs with header seq: 1634 +stamp: + secs: 1734488629 + nsecs: 809862852 +frame_id: '' and grasped_object_mask header seq: 120920 +stamp: + secs: 1734488629 + nsecs: 579621315 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -3059683688 for nearest: 1734488626719776027 - target: 1734488629779459715 at idx 249 +[INFO] [1734488629.996650]: received mask_obs with header seq: 1635 +stamp: + secs: 1734488630 + nsecs: 1130819 +frame_id: '' and grasped_object_mask header seq: 120925 +stamp: + secs: 1734488629 + nsecs: 746153355 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -3259524710 for nearest: 1734488626719776027 - target: 1734488629979300737 at idx 249 +[INFO] [1734488630.197491]: received mask_obs with header seq: 1636 +stamp: + secs: 1734488630 + nsecs: 202248811 +frame_id: '' and grasped_object_mask header seq: 120931 +stamp: + secs: 1734488629 + nsecs: 945994377 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -3459365971 for nearest: 1734488626719776027 - target: 1734488630179141998 at idx 249 +[INFO] [1734488630.397286]: received mask_obs with header seq: 1637 +stamp: + secs: 1734488630 + nsecs: 402032613 +frame_id: '' and grasped_object_mask header seq: 120937 +stamp: + secs: 1734488630 + nsecs: 145835638 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -3659208424 for nearest: 1734488626719776027 - target: 1734488630378984451 at idx 249 +[INFO] [1734488630.597954]: received mask_obs with header seq: 1638 +stamp: + secs: 1734488630 + nsecs: 602582931 +frame_id: '' and grasped_object_mask header seq: 120943 +stamp: + secs: 1734488630 + nsecs: 345676899 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -3859049685 for nearest: 1734488626719776027 - target: 1734488630578825712 at idx 249 +[INFO] [1734488630.797328]: received mask_obs with header seq: 1639 +stamp: + secs: 1734488630 + nsecs: 801355361 +frame_id: '' and grasped_object_mask header seq: 120949 +stamp: + secs: 1734488630 + nsecs: 545518398 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -4058887370 for nearest: 1734488626719776027 - target: 1734488630778663397 at idx 249 +[INFO] [1734488630.996526]: received mask_obs with header seq: 1640 +stamp: + secs: 1734488631 + nsecs: 705718 +frame_id: '' and grasped_object_mask header seq: 120956 +stamp: + secs: 1734488630 + nsecs: 778663397 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -4258722909 for nearest: 1734488626719776027 - target: 1734488630978498936 at idx 249 +[INFO] [1734488631.203860]: received mask_obs with header seq: 1641 +stamp: + secs: 1734488631 + nsecs: 208388328 +frame_id: '' and grasped_object_mask header seq: 120961 +stamp: + secs: 1734488630 + nsecs: 945193529 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -4458558448 for nearest: 1734488626719776027 - target: 1734488631178334475 at idx 249 +[INFO] [1734488631.397352]: received mask_obs with header seq: 1642 +stamp: + secs: 1734488631 + nsecs: 402099132 +frame_id: '' and grasped_object_mask header seq: 120967 +stamp: + secs: 1734488631 + nsecs: 145029068 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -4658393510 for nearest: 1734488626719776027 - target: 1734488631378169537 at idx 249 +[INFO] [1734488631.598122]: received mask_obs with header seq: 1643 +stamp: + secs: 1734488631 + nsecs: 602123737 +frame_id: '' and grasped_object_mask header seq: 120973 +stamp: + secs: 1734488631 + nsecs: 344864368 +frame_id: "camera_color_optical_frame" +[INFO] [1734488631.636867]: Storing episode... +For topic gripper_width: timestamp difference is -4858229287 for nearest: 1734488626719776027 - target: 1734488631578005314 at idx 249 +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1734488631.801016]: received mask_obs with header seq: 1644 +stamp: + secs: 1734488631 + nsecs: 804388046 +frame_id: '' and grasped_object_mask header seq: 120979 +stamp: + secs: 1734488631 + nsecs: 544699192 +frame_id: "camera_color_optical_frame" +[INFO] [1734488632.431817]: Stored episode 3. +[INFO] [1734488632.433711]: Saving video... +[INFO] [1734488632.874029]: Video saved! +Starting episode 3 +[INFO] [1734488632.902666]: resetting environment +[INFO] [1734488632.909836]: cleared current plan +[INFO] [1734488632.918970]: moving to home +[INFO] [1734488635.623441]: reached home +[INFO] [1734488635.634476]: reset action history +[INFO] [1734488638.427099]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1734488722.939590]: Saving video... +[INFO] [1734488722.954511]: Video saved! \ No newline at end of file diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt b/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json b/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c14839418f3e0f46013646a19bb1c41df63f978f --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-18T02:21:24.806252", + "startedAt": "2024-12-18T02:21:24.423927", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3375.3751250000005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4501.58, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.929, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 389.73836517333984 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json b/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fa76ebe975639adcc8249cee6e3cc592287718e2 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7012418043e20ba472ca099eb2fd4bae9cf951ff5f200bf3b8b28bef7c455f17", "size": 860246, "path": "media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4"}, "global_step": 182, "_timestamp": 1734488600.9750466, "_runtime": 116.54287362098694, "_step": 5, "eval/1_eval": {"_type": "video-file", "sha256": "4a30824235f8d3481acdd75328ce66a4db21dec3ebd5b875ba7e3c527508d0f3", "size": 854722, "path": "media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4"}, "eval/num_success": 0.0, "episode": 2.0, "eval/success_rate": 0.0, "eval/2_eval": {"_type": "video-file", "sha256": "fe502a2ffcc54aa7653539234f578d5a1a01cd0bf111c0cb8107c59433150318", "size": 1236066, "path": "media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4"}, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 238}} \ No newline at end of file diff --git a/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log b/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c8b1d6baf60954a5e7bbb795b5d5f3aa840b1771 --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log @@ -0,0 +1,651 @@ +2024-12-17 21:21:24,431 INFO StreamThr :76461 [internal.py:wandb_internal():85] W&B internal server running at pid: 76461, started at: 2024-12-17 21:21:24.431534 +2024-12-17 21:21:24,433 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status +2024-12-17 21:21:24,444 INFO WriterThread:76461 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb +2024-12-17 21:21:24,444 DEBUG SenderThread:76461 [sender.py:send():379] send: header +2024-12-17 21:21:24,453 DEBUG SenderThread:76461 [sender.py:send():379] send: run +2024-12-17 21:21:24,691 INFO SenderThread:76461 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files +2024-12-17 21:21:24,691 INFO SenderThread:76461 [sender.py:_start_run_threads():1188] run started: 78r9r705 with start time 1734488484.432173 +2024-12-17 21:21:24,705 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: check_version +2024-12-17 21:21:24,706 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: check_version +2024-12-17 21:21:24,787 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: run_start +2024-12-17 21:21:24,792 DEBUG HandlerThread:76461 [system_info.py:__init__():26] System info init +2024-12-17 21:21:24,792 DEBUG HandlerThread:76461 [system_info.py:__init__():41] System info init done +2024-12-17 21:21:24,792 INFO HandlerThread:76461 [system_monitor.py:start():194] Starting system monitor +2024-12-17 21:21:24,792 INFO SystemMonitor:76461 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-17 21:21:24,792 INFO HandlerThread:76461 [system_monitor.py:probe():214] Collecting system info +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started cpu monitoring +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started disk monitoring +2024-12-17 21:21:24,793 INFO SystemMonitor:76461 [interfaces.py:start():188] Started gpu monitoring +2024-12-17 21:21:24,794 INFO SystemMonitor:76461 [interfaces.py:start():188] Started memory monitoring +2024-12-17 21:21:24,794 INFO SystemMonitor:76461 [interfaces.py:start():188] Started network monitoring +2024-12-17 21:21:24,806 DEBUG HandlerThread:76461 [system_info.py:probe():152] Probing system +2024-12-17 21:21:24,807 DEBUG HandlerThread:76461 [system_info.py:_probe_git():137] Probing git +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:_probe_git():145] Probing git done +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:probe():200] Probing system done +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:21:24.806252', 'startedAt': '2024-12-18T02:21:24.423927', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3375.3751250000005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4501.58, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.929, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 389.73836517333984}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-17 21:21:24,812 INFO HandlerThread:76461 [system_monitor.py:probe():224] Finished collecting system info +2024-12-17 21:21:24,812 INFO HandlerThread:76461 [system_monitor.py:probe():227] Publishing system info +2024-12-17 21:21:24,812 DEBUG HandlerThread:76461 [system_info.py:_save_code():44] Saving code +2024-12-17 21:21:24,814 DEBUG HandlerThread:76461 [system_info.py:_save_code():67] Saving code done +2024-12-17 21:21:24,814 DEBUG HandlerThread:76461 [system_info.py:_save_patches():84] Saving git patches +2024-12-17 21:21:24,883 DEBUG HandlerThread:76461 [system_info.py:_save_patches():126] Saving git patches done +2024-12-17 21:21:24,883 INFO HandlerThread:76461 [system_monitor.py:probe():229] Finished publishing system info +2024-12-17 21:21:24,886 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:21:24,886 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-17 21:21:24,887 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-17 21:21:24,887 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-17 21:21:25,126 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: python_packages +2024-12-17 21:21:25,126 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: python_packages +2024-12-17 21:21:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:25,128 DEBUG SenderThread:76461 [sender.py:send():379] send: telemetry +2024-12-17 21:21:25,133 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/zf5gfm67-diff.patch +2024-12-17 21:21:25,157 INFO wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/bk0z0q6f-code/FISH/eval_robot.py +2024-12-17 21:21:25,159 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/o7jx2utn-wandb-metadata.json +2024-12-17 21:21:25,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:25,169 INFO HandlerThread:76461 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb" + save: true +} +control { + relay_id: "52ec4a1767a3429cb6d1b8e86029eaaf" +} +_info { + stream_id: "78r9r705" +} + +2024-12-17 21:21:25,169 DEBUG HandlerThread:76461 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-17 21:21:25,170 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:25,277 DEBUG SenderThread:76461 [sender.py:send():379] send: tbrecord +2024-12-17 21:21:25,278 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:21:25,278 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 with policy live +2024-12-17 21:21:25,454 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lobsc1b3-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch +2024-12-17 21:21:25,693 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:25,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH +2024-12-17 21:21:25,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code +2024-12-17 21:21:26,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:27,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:27,692 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:28,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:29,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:30,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:30,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:31,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:31,696 DEBUG SenderThread:76461 [sender.py:send():379] send: config +2024-12-17 21:21:32,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:33,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:33,694 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:34,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:35,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:35,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:36,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:37,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:37,695 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:38,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:39,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:40,126 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:40,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:40,281 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:41,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:41,696 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:42,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:43,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:44,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:45,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:46,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:46,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:47,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:48,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:49,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:50,130 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:51,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:51,699 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:52,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:52,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:53,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:53,699 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:54,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:55,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:21:55,129 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:21:55,173 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:55,700 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:56,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:57,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:57,282 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:21:57,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml +2024-12-17 21:21:57,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:21:58,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:59,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:21:59,701 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:00,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:01,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:01,702 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:02,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:02,562 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:03,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:03,702 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:04,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:05,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:05,703 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:06,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:07,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:07,703 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:07,736 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:08,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:09,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:09,704 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:10,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:10,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:10,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:11,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:11,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:12,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:12,599 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:22:12,599 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 with policy now +2024-12-17 21:22:12,599 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:22:12,600 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:22:12,600 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:22:12,601 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media +2024-12-17 21:22:12,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos +2024-12-17 21:22:12,849 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/0wiijtor-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:13,108 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/2ltzk93y-media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:22:13,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:13,611 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:13,705 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:14,138 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:15,134 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:16,134 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:17,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:17,707 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:18,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:18,625 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:19,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:19,707 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:20,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:21,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:22,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:23,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:24,129 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:24,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:24,794 DEBUG SystemMonitor:76461 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-17 21:22:24,795 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:22:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:25,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:25,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:26,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:27,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:28,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:29,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:29,217 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:30,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:31,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:32,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:33,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:34,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:35,128 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:35,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:36,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:37,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:37,713 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:22:37,978 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/lzwjgswx-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:22:38,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:39,137 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:39,714 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:40,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:40,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:40,228 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:41,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:41,715 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:42,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:43,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:43,715 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:44,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:45,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:45,252 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:45,716 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:46,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:47,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:47,717 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:48,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:49,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:49,718 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:50,140 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:50,256 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:51,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:51,718 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:52,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:53,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:53,719 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:54,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:54,796 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:22:55,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:22:55,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:22:55,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:55,267 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:22:55,720 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:56,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:57,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:57,720 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:22:58,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:59,141 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:22:59,721 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:00,142 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:00,373 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:00,417 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:23:00,417 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:23:00,418 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 with policy now +2024-12-17 21:23:00,418 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:00,418 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:00,418 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:00,673 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/4e5seuc5-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:00,721 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:00,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:00,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:23:00,967 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/akortq61-media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:23:01,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:01,722 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:02,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:03,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:04,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:05,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:05,723 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:06,135 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:06,160 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:07,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:07,724 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:08,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:09,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:10,131 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:10,131 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:10,173 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:11,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:11,232 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:12,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:13,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:14,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:15,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:16,146 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:17,133 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:17,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:18,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:19,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:20,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:21,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:21,571 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:21,571 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:21,571 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:21,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:22,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:23,029 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:23,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:23,729 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:24,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:24,796 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:23:25,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:25,128 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:25,147 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:25,730 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:26,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:27,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:27,731 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:28,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:28,194 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:29,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:29,731 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:30,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:31,148 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:31,732 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:32,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:33,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:33,393 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:33,733 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:34,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:35,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:35,734 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:36,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:37,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:37,734 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:38,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:38,405 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:39,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:39,735 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:40,127 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:40,127 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:40,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:41,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:41,736 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:42,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:43,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:43,409 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:43,737 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:44,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:45,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:45,737 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:46,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:47,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:47,738 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:48,151 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:48,460 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:49,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:49,739 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:50,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:51,151 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:51,740 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:52,150 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:52,900 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:23:52,900 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 with policy now +2024-12-17 21:23:52,900 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:23:52,901 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:23:52,901 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:23:52,901 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:23:53,155 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/evzpq5tf-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:53,155 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:53,741 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:23:53,920 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:54,016 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/jsf3d998-media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:23:54,163 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:54,797 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:23:55,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:23:55,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:23:55,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:56,161 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:57,159 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:57,742 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:23:58,163 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:59,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:23:59,469 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:23:59,743 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:24:00,165 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:01,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:02,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:03,174 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:04,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:05,144 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:05,169 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:06,176 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:07,167 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:08,176 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:09,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:10,136 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:10,136 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:10,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:10,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:11,171 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:12,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:13,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:14,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:15,174 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:16,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:16,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:17,182 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:18,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:19,182 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:20,177 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:21,183 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:22,145 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:22,178 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:23,186 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:24,181 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:24,797 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:24:25,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:25,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:25,185 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:26,179 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:27,184 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:27,208 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:28,189 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:29,188 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:30,185 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:31,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:32,189 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:33,152 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:33,187 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:34,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:35,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:36,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:37,190 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:38,191 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:39,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:39,200 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:40,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:40,132 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:40,201 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:41,203 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:42,204 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:43,205 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:44,203 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:44,227 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:45,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:46,206 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:47,208 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:48,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:49,209 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:50,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:50,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:51,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:52,211 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:53,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:54,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:54,798 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:24:55,132 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:24:55,133 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:24:55,192 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:24:55,213 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:56,214 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:57,210 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:58,213 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:24:59,220 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:00,223 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:01,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:01,219 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:02,216 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:03,216 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:04,218 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:05,220 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:06,226 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:07,149 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:07,223 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:08,228 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:09,225 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:10,142 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:25:10,142 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:25:10,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:11,233 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:11,766 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:12,227 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:12,264 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:13,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:14,236 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:15,256 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:16,257 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:17,264 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:18,160 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:18,246 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:19,253 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:20,253 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:21,248 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:22,252 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:23,047 DEBUG SenderThread:76461 [sender.py:send():379] send: files +2024-12-17 21:25:23,047 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 with policy now +2024-12-17 21:25:23,047 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:25:23,047 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:25:23,048 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:25:23,049 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:23,051 DEBUG SenderThread:76461 [sender.py:send():379] send: exit +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():586] handling exit code: 0 +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():588] handling runtime: 238 +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:23,051 INFO SenderThread:76461 [sender.py:send_exit():594] send defer +2024-12-17 21:25:23,051 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,051 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-17 21:25:23,052 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 1 +2024-12-17 21:25:23,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-17 21:25:23,052 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-17 21:25:23,052 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 2 +2024-12-17 21:25:23,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [system_monitor.py:finish():203] Stopping system monitor +2024-12-17 21:25:23,052 DEBUG SystemMonitor:76461 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-17 21:25:23,052 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined cpu monitor +2024-12-17 21:25:23,052 DEBUG SystemMonitor:76461 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-17 21:25:23,053 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined disk monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined gpu monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined memory monitor +2024-12-17 21:25:23,096 INFO HandlerThread:76461 [interfaces.py:finish():200] Joined network monitor +2024-12-17 21:25:23,096 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,096 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-17 21:25:23,096 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 3 +2024-12-17 21:25:23,097 DEBUG SenderThread:76461 [sender.py:send():379] send: stats +2024-12-17 21:25:23,097 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,097 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-17 21:25:23,097 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:23,097 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-17 21:25:23,097 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 4 +2024-12-17 21:25:23,097 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:23,097 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-17 21:25:23,269 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/rrl4ps4z-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:23,770 INFO Thread-12 :76461 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:23,771 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval +2024-12-17 21:25:24,015 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/ipbifp7e-media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:28,586 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:28,586 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:28,586 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,587 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-17 21:25:28,587 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 5 +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:28,587 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send():379] send: history +2024-12-17 21:25:28,587 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-17 21:25:28,587 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:28,588 DEBUG SenderThread:76461 [sender.py:send():379] send: summary +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:25:28,588 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-17 21:25:28,588 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 6 +2024-12-17 21:25:28,588 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,588 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-17 21:25:28,589 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 7 +2024-12-17 21:25:28,589 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:25:28,589 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:28,589 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-17 21:25:28,589 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:28,589 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-17 21:25:28,772 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:29,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: keepalive +2024-12-17 21:25:29,052 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:29,669 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 8 +2024-12-17 21:25:29,669 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:29,669 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:29,669 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-17 21:25:29,670 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:29,670 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-17 21:25:29,670 INFO SenderThread:76461 [job_builder.py:build():440] Attempting to build job artifact +2024-12-17 21:25:29,672 INFO SenderThread:76461 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-17 21:25:29,686 INFO SenderThread:76461 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-17 21:25:29,700 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 9 +2024-12-17 21:25:29,700 DEBUG SenderThread:76461 [sender.py:send():379] send: artifact +2024-12-17 21:25:29,700 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:29,702 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-17 21:25:29,772 INFO Thread-12 :76461 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:30,053 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:30,189 INFO wandb-upload_1:76461 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpat7ok6vh +2024-12-17 21:25:30,320 INFO wandb-upload_2:76461 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu793frvf +2024-12-17 21:25:30,360 INFO wandb-upload_0:76461 [upload_job.py:push():88] Uploaded file /tmp/tmp41i79nsx/wandb-job.json +2024-12-17 21:25:30,913 INFO SenderThread:76461 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'versionIndex': 378}}} +2024-12-17 21:25:30,913 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:30,913 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-17 21:25:30,913 INFO SenderThread:76461 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml config.yaml +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json wandb-summary.json +2024-12-17 21:25:31,773 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt requirements.txt +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-metadata.json wandb-metadata.json +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log output.log +2024-12-17 21:25:31,774 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/diff.patch diff.patch +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_3_25445a41b5bbda0ed7e2.mp4 +2024-12-17 21:25:31,777 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 media/videos/eval/2_eval_2_fe502a2ffcc54aa76535.mp4 +2024-12-17 21:25:31,778 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 media/videos/eval/1_eval_1_4a30824235f8d3481acd.mp4 +2024-12-17 21:25:31,779 INFO SenderThread:76461 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 media/videos/eval/0_eval_0_7012418043e20ba472ca.mp4 +2024-12-17 21:25:31,780 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 10 +2024-12-17 21:25:31,780 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:31,782 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:31,782 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-17 21:25:31,782 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:31,782 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-17 21:25:31,782 INFO SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:25:31,949 INFO wandb-upload_0:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/wandb-summary.json +2024-12-17 21:25:31,988 INFO wandb-upload_1:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/config.yaml +2024-12-17 21:25:31,990 INFO wandb-upload_3:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/requirements.txt +2024-12-17 21:25:32,014 INFO wandb-upload_2:76461 [upload_job.py:push():130] Uploaded file /tmp/tmptw3lk4z3wandb/kg0vdiiy-events.out.tfevents.1734488485.leonmkim-ROG-Strix-G15CS-G15CS.76272.0 +2024-12-17 21:25:32,036 INFO wandb-upload_4:76461 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/files/output.log +2024-12-17 21:25:32,053 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,054 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,237 INFO Thread-11 (_thread_body):76461 [sender.py:transition_state():614] send defer: 11 +2024-12-17 21:25:32,237 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,237 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-17 21:25:32,237 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,237 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-17 21:25:32,237 INFO SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher +2024-12-17 21:25:32,238 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 12 +2024-12-17 21:25:32,238 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,238 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-17 21:25:32,238 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,238 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-17 21:25:32,238 INFO SenderThread:76461 [file_stream.py:finish():601] file stream finish called +2024-12-17 21:25:32,430 INFO SenderThread:76461 [file_stream.py:finish():605] file stream finish is done +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 13 +2024-12-17 21:25:32,430 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,430 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-17 21:25:32,430 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-17 21:25:32,430 INFO SenderThread:76461 [sender.py:transition_state():614] send defer: 14 +2024-12-17 21:25:32,430 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:25:32,431 INFO HandlerThread:76461 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send():379] send: final +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send():379] send: footer +2024-12-17 21:25:32,431 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: defer +2024-12-17 21:25:32,431 INFO SenderThread:76461 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-17 21:25:32,431 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,431 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: server_info +2024-12-17 21:25:32,432 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:25:32,432 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,432 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: server_info +2024-12-17 21:25:32,432 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: get_summary +2024-12-17 21:25:32,434 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-17 21:25:32,434 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:25:32,507 DEBUG SenderThread:76461 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-17 21:25:32,507 INFO MainThread:76461 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-17 21:25:32,507 DEBUG HandlerThread:76461 [handler.py:handle_request():158] handle_request: shutdown +2024-12-17 21:25:32,507 INFO HandlerThread:76461 [handler.py:finish():882] shutting down handler +2024-12-17 21:25:33,432 INFO WriterThread:76461 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb +2024-12-17 21:25:33,507 INFO SenderThread:76461 [sender.py:finish():1615] shutting down sender +2024-12-17 21:25:33,507 INFO SenderThread:76461 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:25:33,507 INFO SenderThread:76461 [file_pusher.py:join():175] waiting for file pusher diff --git a/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log b/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..769b44f57a072605182fa99537bc3e7590c8d14c --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Configure stats pid to 76272 +2024-12-17 21:21:24,425 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/settings +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug.log +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/wandb/run-20241217_212124-78r9r705/logs/debug-internal.log +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():569] calling init triggers +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118'} +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():619] starting backend +2024-12-17 21:21:24,426 INFO MainThread:76272 [wandb_init.py:init():623] setting up manager +2024-12-17 21:21:24,430 INFO MainThread:76272 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-17 21:21:24,431 INFO MainThread:76272 [wandb_init.py:init():631] backend started and connected +2024-12-17 21:21:24,443 INFO MainThread:76272 [wandb_init.py:init():720] updated telemetry +2024-12-17 21:21:24,450 INFO MainThread:76272 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-17 21:21:24,705 INFO MainThread:76272 [wandb_run.py:_on_init():2435] communicating current version +2024-12-17 21:21:24,776 INFO MainThread:76272 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-17 21:21:24,779 INFO MainThread:76272 [wandb_init.py:init():804] starting run threads in backend +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_console_start():2413] atexit reg +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-17 21:21:25,127 INFO MainThread:76272 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-17 21:21:25,129 INFO MainThread:76272 [wandb_init.py:init():847] run started, returning control to user process +2024-12-17 21:21:25,129 INFO MainThread:76272 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/212118/tb, True +2024-12-17 21:21:31,696 INFO MainThread:76272 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2024-12-17 21:25:33,510 WARNING MsgRouterThr:76272 [router.py:message_loop():77] message_loop has been closed diff --git a/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb b/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0613f4218dadd5033a31041d908aef2ae593e95c --- /dev/null +++ b/212118/wandb/run-20241217_212124-78r9r705/run-78r9r705.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d081babd0cb3a79eceffd7c8022f97e7e0cbbc6ef1dc6f783b0b53f328279a13 +size 159910 diff --git a/215239/.hydra/config.yaml b/215239/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7462739dbe7df1e4371890667e7c6210ec95aee --- /dev/null +++ b/215239/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '523_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/215239/.hydra/hydra.yaml b/215239/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81ee9b42d20b86d4935bf6ff3571bda0e8c4e801 --- /dev/null +++ b/215239/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215239 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/215239/.hydra/overrides.yaml b/215239/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/215239/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/215239/eval_policy.log b/215239/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..a84008123ab8cbdda3c4425221296bd7a560a09b --- /dev/null +++ b/215239/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-17 21:52:39,536][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-17 21:52:39,540][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-17 21:52:41,548][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/215240/.hydra/config.yaml b/215240/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7462739dbe7df1e4371890667e7c6210ec95aee --- /dev/null +++ b/215240/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '523_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/215240/.hydra/hydra.yaml b/215240/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..886a009750fece2d18eea962cab624e0519f949e --- /dev/null +++ b/215240/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/215240/.hydra/overrides.yaml b/215240/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/215240/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/215240/episode_rosbags/aligned_depth_to_color_K.npy b/215240/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/215240/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/215240/episode_rosbags/cam_tf_world.npy b/215240/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/215240/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/215240/episode_rosbags/color_K.npy b/215240/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/215240/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/215240/episode_rosbags/depth_K.npy b/215240/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/215240/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag b/215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..8ec1af18140430ea5e87be382a7d011501c92f50 --- /dev/null +++ b/215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:327377ba0f4c35a511dcbba69c5edef66fb7bf0f9f47b8b143509addeb82a43c +size 1410388741 diff --git a/215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag b/215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..54a9fd69b6d21617c3774ce7559192fe9e7286db --- /dev/null +++ b/215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3957ac7f81c2697bc53a35e8836a90828121ea8cc68cf2bb77f09f9fa2beb2 +size 1407810142 diff --git a/215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag b/215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..ce16dc25912fb7321c4aeb1b89c270a095df66ed --- /dev/null +++ b/215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f69d01cd59a9f5ee5ec7cc9d497a7ad296881f66699eb2f60da187af77832800 +size 1415524052 diff --git a/215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag b/215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..5f7e11a1af71d906752db398fcc30acc877fa6a6 --- /dev/null +++ b/215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1820cc33e77bc615657e09ae62052b6c782ac3d03d0e3cf99d214a4e3bd0b7ea +size 992970612 diff --git a/215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag b/215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..dba4ccbb9d9c015e731bd493a91a64ffa1f4f3a9 --- /dev/null +++ b/215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76d404d7beea0e3a05dab1d4e625f729343a1bd78980e1cfda50c0a65d3c213e +size 1412037126 diff --git a/215240/eval_robot.log b/215240/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6fc589bc476eb7a8f21223e1196f2bacb370b084 --- /dev/null +++ b/215240/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-17 21:52:40,235][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-17 21:52:40,238][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/215240/eval_video/0_eval.mp4 b/215240/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a151ab7bca71294982bf5fd0c7130f5b10191f5 --- /dev/null +++ b/215240/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbd1eabcbea39a4495283b2f6184ef4a084c2c15f963d7e5faa8bbd22cbd06b6 +size 1175932 diff --git a/215240/eval_video/1_eval.mp4 b/215240/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b44a829e0e15c995dbc546963f30c2335af1dea3 --- /dev/null +++ b/215240/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42cf14600b8d2071849cb7efe24d11a32495dab3954997e2c2a53233f5841b6b +size 1179891 diff --git a/215240/eval_video/2_eval.mp4 b/215240/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1e0ba6f610ad166bc12c317f4e88a8b175eb880 --- /dev/null +++ b/215240/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5329c53c34465295af0a20968ecc659fb9b1f59c569d2d70bc8ca0b2b92cb0b7 +size 1184804 diff --git a/215240/eval_video/3_eval.mp4 b/215240/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d6938282ded192172dbf472c7c87a73e7459186f --- /dev/null +++ b/215240/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f27b273940874a00d0beb9ee861cc2707f80e7a7361acb0344fc73e8ad9ba8 +size 929140 diff --git a/215240/eval_video/4_eval.mp4 b/215240/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..192b5223ba1631747fd000cab474d55bca96c0f4 --- /dev/null +++ b/215240/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08fea80871bf35a0fb30f5931132d38f7ca937e097e8e5057847c2ba0b1ff3ee +size 1194519 diff --git a/215240/tb/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 b/215240/tb/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 new file mode 100644 index 0000000000000000000000000000000000000000..305916958c1376e678db59e8fa855f9ae96967b1 --- /dev/null +++ b/215240/tb/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcc4c66e5f4885c82d7744b80e8f0cdf222bc468f188ceea469f34d760223a7 +size 1103 diff --git a/215240/wandb/debug-internal.log b/215240/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d8c3cc254357ef2c26fd0bc3d5b4da4209dde30f --- /dev/null +++ b/215240/wandb/debug-internal.log @@ -0,0 +1,753 @@ +2024-12-17 21:52:46,130 INFO StreamThr :78001 [internal.py:wandb_internal():85] W&B internal server running at pid: 78001, started at: 2024-12-17 21:52:46.129837 +2024-12-17 21:52:46,131 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status +2024-12-17 21:52:46,144 INFO WriterThread:78001 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb +2024-12-17 21:52:46,144 DEBUG SenderThread:78001 [sender.py:send():379] send: header +2024-12-17 21:52:46,152 DEBUG SenderThread:78001 [sender.py:send():379] send: run +2024-12-17 21:52:46,475 INFO SenderThread:78001 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files +2024-12-17 21:52:46,475 INFO SenderThread:78001 [sender.py:_start_run_threads():1188] run started: g6nsr00w with start time 1734490366.130536 +2024-12-17 21:52:46,494 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: check_version +2024-12-17 21:52:46,494 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: check_version +2024-12-17 21:52:46,540 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: run_start +2024-12-17 21:52:46,544 DEBUG HandlerThread:78001 [system_info.py:__init__():26] System info init +2024-12-17 21:52:46,544 DEBUG HandlerThread:78001 [system_info.py:__init__():41] System info init done +2024-12-17 21:52:46,544 INFO HandlerThread:78001 [system_monitor.py:start():194] Starting system monitor +2024-12-17 21:52:46,545 INFO SystemMonitor:78001 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-17 21:52:46,545 INFO HandlerThread:78001 [system_monitor.py:probe():214] Collecting system info +2024-12-17 21:52:46,545 INFO SystemMonitor:78001 [interfaces.py:start():188] Started cpu monitoring +2024-12-17 21:52:46,546 INFO SystemMonitor:78001 [interfaces.py:start():188] Started disk monitoring +2024-12-17 21:52:46,546 INFO SystemMonitor:78001 [interfaces.py:start():188] Started gpu monitoring +2024-12-17 21:52:46,547 INFO SystemMonitor:78001 [interfaces.py:start():188] Started memory monitoring +2024-12-17 21:52:46,547 INFO SystemMonitor:78001 [interfaces.py:start():188] Started network monitoring +2024-12-17 21:52:46,558 DEBUG HandlerThread:78001 [system_info.py:probe():152] Probing system +2024-12-17 21:52:46,560 DEBUG HandlerThread:78001 [system_info.py:_probe_git():137] Probing git +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_info.py:_probe_git():145] Probing git done +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_info.py:probe():200] Probing system done +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:52:46.558919', 'startedAt': '2024-12-18T02:52:46.119968', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3572.842125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4517.241, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4536.345, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 392.8499336242676}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-17 21:52:46,565 INFO HandlerThread:78001 [system_monitor.py:probe():224] Finished collecting system info +2024-12-17 21:52:46,565 INFO HandlerThread:78001 [system_monitor.py:probe():227] Publishing system info +2024-12-17 21:52:46,565 DEBUG HandlerThread:78001 [system_info.py:_save_code():44] Saving code +2024-12-17 21:52:46,567 DEBUG HandlerThread:78001 [system_info.py:_save_code():67] Saving code done +2024-12-17 21:52:46,567 DEBUG HandlerThread:78001 [system_info.py:_save_patches():84] Saving git patches +2024-12-17 21:52:46,676 DEBUG HandlerThread:78001 [system_info.py:_save_patches():126] Saving git patches done +2024-12-17 21:52:46,677 INFO HandlerThread:78001 [system_monitor.py:probe():229] Finished publishing system info +2024-12-17 21:52:46,680 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-17 21:52:46,907 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: python_packages +2024-12-17 21:52:46,907 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: python_packages +2024-12-17 21:52:46,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:52:46,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:52:46,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:46,910 INFO HandlerThread:78001 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/tb" + save: true +} +control { + relay_id: "394677d9b6aa4385a9b0ebca7a0e61ef" +} +_info { + stream_id: "g6nsr00w" +} + +2024-12-17 21:52:46,911 DEBUG HandlerThread:78001 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-17 21:52:46,935 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/cmvh8ay6-wandb-metadata.json +2024-12-17 21:52:46,944 INFO wandb-upload_1:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/iupp8rxj-code/FISH/eval_robot.py +2024-12-17 21:52:46,953 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/fmlykv68-diff.patch +2024-12-17 21:52:47,056 DEBUG SenderThread:78001 [sender.py:send():379] send: telemetry +2024-12-17 21:52:47,056 DEBUG SenderThread:78001 [sender.py:send():379] send: tbrecord +2024-12-17 21:52:47,057 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:52:47,057 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 with policy live +2024-12-17 21:52:47,234 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/xsh0mhuy-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code +2024-12-17 21:52:47,478 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH +2024-12-17 21:52:47,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:48,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:49,476 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:49,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:50,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:51,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:52:51,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:52,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:53,720 DEBUG SenderThread:78001 [sender.py:send():379] send: config +2024-12-17 21:52:53,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:54,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:55,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:55,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:56,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:52:56,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:57,478 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:57,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:58,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:59,479 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:59,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:00,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:01,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:01,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:01,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:02,060 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:02,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:03,480 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:03,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:04,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:05,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:06,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:07,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:07,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:08,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:09,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:10,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:11,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:12,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:12,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:13,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:14,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:15,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:16,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:16,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:16,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:17,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:18,033 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:18,486 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml +2024-12-17 21:53:18,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:19,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:20,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:21,487 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:21,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:22,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:23,488 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:23,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:23,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:24,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:25,488 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:25,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:26,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:27,489 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:27,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:28,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:29,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:29,490 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:29,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:30,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:31,490 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:31,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:31,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:31,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:32,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:33,491 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:33,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:34,165 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:34,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:35,491 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:35,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:36,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:37,492 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:37,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:38,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:39,330 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:39,492 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:39,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:40,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:41,493 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:41,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:42,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:43,494 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:43,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:44,529 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:44,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:45,494 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:45,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:46,548 DEBUG SystemMonitor:78001 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-17 21:53:46,548 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:53:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:46,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:46,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:47,495 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:47,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:48,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:49,495 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:49,548 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:49,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:50,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:51,496 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:51,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:52,319 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:53:52,319 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 with policy now +2024-12-17 21:53:52,319 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:53:52,320 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:53:52,320 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:53:52,320 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:53:52,496 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media +2024-12-17 21:53:52,542 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rdsdzo9c-media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,775 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/ydvne3ig-media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:53,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:53,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:54,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:55,331 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:55,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:56,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:57,498 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:57,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:58,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:59,499 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:59,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:00,689 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:00,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:01,500 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:01,741 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/kizcyfj9-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:01,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:01,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:01,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:02,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:03,500 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:03,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:04,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:05,834 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:05,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:06,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:07,502 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:07,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:08,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:09,502 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:09,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:10,869 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:10,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:11,503 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:11,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:12,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:13,504 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:13,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:14,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:15,505 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:15,871 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:15,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:16,549 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:54:16,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:16,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:16,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:17,505 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:17,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:18,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:19,506 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:19,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:20,871 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:20,922 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:21,507 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:21,922 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:22,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:23,507 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:23,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:24,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:25,508 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:25,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:26,078 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:26,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:27,509 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:27,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:28,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:29,510 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:29,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:30,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:31,274 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:31,510 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:31,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:31,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:31,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:32,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:33,409 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:54:33,409 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 with policy now +2024-12-17 21:54:33,412 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:54:33,412 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:54:33,413 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:54:33,413 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,512 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:54:33,641 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/hvxa2wmt-media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,894 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/hllcqyuf-media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,930 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:34,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:35,512 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:35,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:36,427 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:36,933 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:37,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:38,933 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:39,513 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:39,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:40,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:41,514 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:41,861 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:41,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:42,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:43,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:44,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:45,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:46,549 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:54:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:46,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:46,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:47,040 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:47,312 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:54:47,312 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:54:47,312 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:47,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:48,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:49,517 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:49,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:50,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:51,517 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:51,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:52,197 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:52,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:53,518 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:53,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:54,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:55,519 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:55,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:56,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:57,220 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:57,520 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:57,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:58,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:59,520 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:59,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:00,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:01,521 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:01,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:01,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:01,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:02,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:02,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:03,522 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:03,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:04,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:05,523 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:05,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:06,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:07,423 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:07,523 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:07,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:08,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:09,524 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:09,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:10,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:11,525 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:11,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:12,610 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:12,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:13,525 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:13,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:14,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:15,526 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:15,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:16,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:55:16,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:16,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:16,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:17,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:17,610 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:17,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:18,608 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:55:18,608 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 with policy now +2024-12-17 21:55:18,609 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:55:18,609 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:18,609 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:18,610 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:18,851 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rquo0y7d-media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:18,941 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:19,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:19,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:19,530 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:19,530 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:55:19,800 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/11petlc4-media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:19,941 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:20,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:21,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:22,944 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:23,285 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:23,531 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:23,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:24,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:25,531 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:25,943 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:26,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:27,532 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:27,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:28,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:28,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:29,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:30,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:31,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:31,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:31,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:32,154 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:32,154 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:32,155 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:32,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:32,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:55:32,777 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/2vpbc8if-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:55:32,954 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:33,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:33,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:34,645 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:34,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:35,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:36,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:37,535 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:37,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:38,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:39,536 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:39,665 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:39,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:40,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:41,537 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:41,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:42,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:43,537 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:43,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:44,867 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:44,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:45,538 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:45,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:46,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:55:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:46,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:46,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:47,539 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:47,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:48,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:49,540 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:49,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:50,059 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:50,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:51,541 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:51,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:52,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:53,541 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:53,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:54,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:55,256 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:55,542 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:55,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:56,842 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:55:56,842 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 with policy now +2024-12-17 21:55:56,847 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:55:56,847 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:56,847 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:56,848 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:56,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:57,057 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/i1alqjlt-media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:55:57,793 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/zvcyis0o-media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:58,966 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:59,967 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:00,295 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:00,968 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:01,544 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:01,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:01,912 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:01,971 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:02,965 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:03,545 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:03,969 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:04,971 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:05,977 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:06,157 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:06,972 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:07,974 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:08,982 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:09,976 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:10,980 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:11,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:11,983 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:12,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:13,986 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:14,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:15,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:16,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:56:16,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:16,916 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:16,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:17,034 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:17,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:18,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:56:18,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:19,211 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:56:19,211 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:56:19,212 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:56:19,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:19,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:56:19,985 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:20,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:21,551 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:21,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:22,275 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:22,986 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:23,552 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:23,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:24,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:25,552 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:25,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:26,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:27,334 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:27,553 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:27,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:28,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:29,554 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:29,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:30,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:31,555 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:31,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:31,911 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:31,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:32,337 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:32,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:33,556 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:33,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:34,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:35,556 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:35,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:36,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:37,525 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:37,557 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:37,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:38,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:39,561 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:39,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:40,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:41,566 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:41,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:42,525 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:42,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:43,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:44,568 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:44,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:45,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:46,551 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:56:46,569 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:46,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:46,911 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:46,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:47,527 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:47,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:48,570 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:48,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:49,995 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:50,434 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:56:50,435 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 with policy now +2024-12-17 21:56:50,437 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:56:50,437 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:56:50,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:56:50,438 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:50,572 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:56:50,640 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rw2u2d5b-media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:50,894 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/z1u4c5tn-media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:51,006 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:52,023 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:52,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:52,929 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:52,999 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:54,021 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:55,000 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:56,013 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:56,573 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:57,001 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:57,973 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:58,012 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:58,573 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:59,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:00,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:01,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:01,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:57:01,921 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:57:02,015 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:03,016 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:03,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:04,016 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:04,437 DEBUG SenderThread:78001 [sender.py:send():379] send: exit +2024-12-17 21:57:04,437 INFO SenderThread:78001 [sender.py:send_exit():586] handling exit code: 0 +2024-12-17 21:57:04,437 INFO SenderThread:78001 [sender.py:send_exit():588] handling runtime: 257 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_exit():594] send defer +2024-12-17 21:57:04,438 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,438 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-17 21:57:04,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 1 +2024-12-17 21:57:04,438 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,438 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-17 21:57:04,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 2 +2024-12-17 21:57:04,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [system_monitor.py:finish():203] Stopping system monitor +2024-12-17 21:57:04,439 DEBUG SystemMonitor:78001 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined cpu monitor +2024-12-17 21:57:04,439 DEBUG SystemMonitor:78001 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined disk monitor +2024-12-17 21:57:04,486 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined gpu monitor +2024-12-17 21:57:04,486 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined memory monitor +2024-12-17 21:57:04,487 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined network monitor +2024-12-17 21:57:04,487 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,487 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-17 21:57:04,487 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 3 +2024-12-17 21:57:04,487 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:57:04,487 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,487 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-17 21:57:04,488 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,488 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-17 21:57:04,488 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 4 +2024-12-17 21:57:04,488 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,488 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-17 21:57:04,575 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:04,576 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:04,810 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/ypj7c7k4-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:06,576 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:10,491 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: keepalive +2024-12-17 21:57:10,491 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-17 21:57:10,492 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 5 +2024-12-17 21:57:10,492 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:57:10,492 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:57:10,492 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,493 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-17 21:57:10,493 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,493 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:10,493 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:57:10,494 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,497 DEBUG SenderThread:78001 [sender.py:send():379] send: summary +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,497 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 6 +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,498 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-17 21:57:10,498 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 7 +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,498 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-17 21:57:10,498 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-17 21:57:10,577 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:11,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:13,429 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 8 +2024-12-17 21:57:13,429 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:13,429 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:13,429 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-17 21:57:13,430 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:13,430 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-17 21:57:13,430 INFO SenderThread:78001 [job_builder.py:build():440] Attempting to build job artifact +2024-12-17 21:57:13,432 INFO SenderThread:78001 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-17 21:57:13,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:13,451 INFO SenderThread:78001 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-17 21:57:13,461 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 9 +2024-12-17 21:57:13,462 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:13,462 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:13,462 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-17 21:57:13,462 DEBUG SenderThread:78001 [sender.py:send():379] send: artifact +2024-12-17 21:57:13,579 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:14,015 INFO wandb-upload_0:78001 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkt9zbjlr +2024-12-17 21:57:14,015 INFO wandb-upload_1:78001 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxnpvbcq3 +2024-12-17 21:57:14,229 INFO wandb-upload_2:78001 [upload_job.py:push():88] Uploaded file /tmp/tmpz2cro5k3/wandb-job.json +2024-12-17 21:57:14,440 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:14,830 INFO SenderThread:78001 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzMzNjA0Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'versionIndex': 379}}} +2024-12-17 21:57:14,830 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:14,830 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-17 21:57:14,830 INFO SenderThread:78001 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-17 21:57:15,580 INFO SenderThread:78001 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml config.yaml +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json wandb-summary.json +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt requirements.txt +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json wandb-metadata.json +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log output.log +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch diff.patch +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 10 +2024-12-17 21:57:15,590 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:15,590 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:15,590 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-17 21:57:15,590 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:15,590 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:57:15,773 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml +2024-12-17 21:57:15,794 INFO wandb-upload_3:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt +2024-12-17 21:57:15,812 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:15,813 INFO wandb-upload_1:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:16,013 INFO Thread-11 (_thread_body):78001 [sender.py:transition_state():614] send defer: 11 +2024-12-17 21:57:16,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,014 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-17 21:57:16,014 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,014 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-17 21:57:16,014 INFO SenderThread:78001 [file_pusher.py:join():175] waiting for file pusher +2024-12-17 21:57:16,017 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 12 +2024-12-17 21:57:16,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,017 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-17 21:57:16,017 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,017 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-17 21:57:16,018 INFO SenderThread:78001 [file_stream.py:finish():601] file stream finish called +2024-12-17 21:57:16,223 INFO SenderThread:78001 [file_stream.py:finish():605] file stream finish is done +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 13 +2024-12-17 21:57:16,223 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,223 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-17 21:57:16,223 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 14 +2024-12-17 21:57:16,223 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,223 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send():379] send: final +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send():379] send: footer +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,224 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-17 21:57:16,224 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:16,224 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: server_info +2024-12-17 21:57:16,225 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:16,225 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: server_info +2024-12-17 21:57:16,226 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: get_summary +2024-12-17 21:57:16,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-17 21:57:16,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:16,292 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-17 21:57:16,293 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: shutdown +2024-12-17 21:57:16,293 INFO HandlerThread:78001 [handler.py:finish():882] shutting down handler +2024-12-17 21:57:17,225 INFO WriterThread:78001 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb +2024-12-17 21:57:17,293 INFO SenderThread:78001 [sender.py:finish():1615] shutting down sender +2024-12-17 21:57:17,293 INFO SenderThread:78001 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:57:17,293 INFO SenderThread:78001 [file_pusher.py:join():175] waiting for file pusher diff --git a/215240/wandb/debug.log b/215240/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0bc576de422da9b1767af1fc9f6047cc614a3e8c --- /dev/null +++ b/215240/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Configure stats pid to 77808 +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/settings +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug.log +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug-internal.log +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():569] calling init triggers +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240'} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():619] starting backend +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():623] setting up manager +2024-12-17 21:52:46,129 INFO MainThread:77808 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-17 21:52:46,130 INFO MainThread:77808 [wandb_init.py:init():631] backend started and connected +2024-12-17 21:52:46,143 INFO MainThread:77808 [wandb_init.py:init():720] updated telemetry +2024-12-17 21:52:46,149 INFO MainThread:77808 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-17 21:52:46,493 INFO MainThread:77808 [wandb_run.py:_on_init():2435] communicating current version +2024-12-17 21:52:46,533 INFO MainThread:77808 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-17 21:52:46,534 INFO MainThread:77808 [wandb_init.py:init():804] starting run threads in backend +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_console_start():2413] atexit reg +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-17 21:52:46,909 INFO MainThread:77808 [wandb_init.py:init():847] run started, returning control to user process +2024-12-17 21:52:46,910 INFO MainThread:77808 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/tb, True +2024-12-17 21:52:53,719 INFO MainThread:77808 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2024-12-17 21:57:17,307 WARNING MsgRouterThr:77808 [router.py:message_loop():77] message_loop has been closed diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py b/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml b/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db42f6f138a89685c64ed749b31af84ada8181de --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml @@ -0,0 +1,891 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '523_0' +wandb_notes: + desc: null + value: 523_0_req_603_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - fowlers_twodim +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1734490366 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch b/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..07ef51bba00eb07cad49fbccb9c4c98ad2796b36 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch @@ -0,0 +1,195 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb0a413..6b5d639 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -107,7 +107,7 @@ num_valid_demos: null + val_num_groups: 3 + + # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act' +-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act' ++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act' + expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}' + + expert_dataset: '${expert_dataset_dirpath}/demos.zarr' +@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} +-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full 32 demos +-# wandb_run_id: "265625" # RGBD history seed 0 +-# wandb_run_id: "266370_1" # RGBD history seed 1 +-# wandb_run_id: "266370_2" # RGBD history seed 2 +-# wandb_run_id: "266382_4" # RGBD history seed 4 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0 +-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1 +-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2 +- +-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0 +-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1 +-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2 +-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3 +-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4 +- +-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0 +-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1 +-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2 +-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3 +-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 +- +-# 8 held out demos (26 demos) +-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0 +-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1 +-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2 +- +-# 8 held out demos (26 demos) +-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0 +-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1 +-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0 +-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1 +-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2 +-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4 +- +-# 16 held out demos (16 demos) +-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0 +-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1 +-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2 +-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4 +- +- +- +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full dataset +-# 6 step action history +-# wandb_run_id: '250358_0' # RGBD+act history +-# wandb_run_id: '250371_0' # RGBD+mask+act history +-# wandb_run_id: '250366_0' # RGBD+contact+act history +-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250366_0' # RGBD+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 16 held out demos +-# wandb_run_id: '250362_0' # RGBD+act history +-# wandb_run_id: '250373_0' # RGBD+mask+act history +-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250368_0' # RGBD+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 32 held out demos +-# wandb_run_id: '250365_0' # RGBD+act history +-# wandb_run_id: '250375_0' # RGBD+mask+act history +-wandb_run_id: '250380_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250370_0' # RGBD+contact+act history +-# 4 step action history +-# wandb_run_id: '264772_0' # RGBD+act history +-# wandb_run_id: '264775_0' # RGBD+mask+act history +-# wandb_run_id: '264776_0' # RGBD+contact+act history +-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history +- +-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# 16 held out demos +-# wandb_run_id: '250362_0' # RGBD+act history +-# wandb_run_id: '250373_0' # RGBD+mask+act history +-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history +-# wandb_run_id: '250368_0' # RGBD+contact+act history ++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# 14 demos ++ ++# RGBD+act history ++# wandb_run_id: '497_0' # german ++# wandb_run_id: '505_0' # hbm ++# wandb_run_id: '506_0' # lady ++# wandb_run_id: '507_0' # greece ++# wandb_run_id: '508_0' # fowlers ++# wandb_run_id: '509_0' # lib ++# wandb_run_id: '510_0' # modelsys ++# wandb_run_id: '511_0' # electrodyn ++ ++# RGBD+mask+act history ++# wandb_run_id: '512_0' # german ++# wandb_run_id: '520_0' # hbm ++# wandb_run_id: '521_0' # lady ++# wandb_run_id: '522_0' # greece ++wandb_run_id: '523_0' # fowlers ++# wandb_run_id: '524_0' # lib ++# wandb_run_id: '525_0' # modelsys ++# wandb_run_id: '526_0' # electrodyn + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 32 held out demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 9406a46..0f225ec 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists(): + local_checkpoint_root_dir.mkdir(parents=True) + + # Download the model checkpoints +-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') + remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser() + remote_username = 'leonmkim' +-remote_host = 'halley' ++remote_host = 'grasp-login1' + + #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 ++# run_id_list = [ ++# '512_0', # german ++# '520_0', # hbm ++# '521_0', # lady ++# '522_0', # greece ++# '523_0', # fowlers ++# '524_0', # lib ++# '525_0', # modelsys ++# '526_0', # electrodyn ++# ] ++ + # run_id_list = [ + # '265638', + # '265637', +@@ -75,6 +87,17 @@ remote_host = 'halley' + # ] + + # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 ++# run_id_list = [ ++# '497_0', # german ++# '505_0', # hbm ++# '506_0', # lady ++# '507_0', # greece ++# '508_0', # fowlers ++# '509_0', # lib ++# '510_0', # modelsys ++# '511_0', # electrodyn ++# ] ++ + # run_id_list = [ + # '266382_4', + # '266370_2', +@@ -116,13 +139,13 @@ remote_host = 'halley' + # ] + + # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +-run_id_list = [ +- # '266384_4,# not yet available +- '266372_1', +- '266372_2', +- '265627', +- '265620', +-] ++# run_id_list = [ ++# # '266384_4,# not yet available ++# '266372_1', ++# '266372_2', ++# '265627', ++# '265620', ++# ] + checkpoint_epoch = 12000 + + # use subprocess to download the model checkpoints in parallel diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a151ab7bca71294982bf5fd0c7130f5b10191f5 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbd1eabcbea39a4495283b2f6184ef4a084c2c15f963d7e5faa8bbd22cbd06b6 +size 1175932 diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b44a829e0e15c995dbc546963f30c2335af1dea3 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42cf14600b8d2071849cb7efe24d11a32495dab3954997e2c2a53233f5841b6b +size 1179891 diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e1e0ba6f610ad166bc12c317f4e88a8b175eb880 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5329c53c34465295af0a20968ecc659fb9b1f59c569d2d70bc8ca0b2b92cb0b7 +size 1184804 diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d6938282ded192172dbf472c7c87a73e7459186f --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f27b273940874a00d0beb9ee861cc2707f80e7a7361acb0344fc73e8ad9ba8 +size 929140 diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..192b5223ba1631747fd000cab474d55bca96c0f4 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08fea80871bf35a0fb30f5931132d38f7ca937e097e8e5057847c2ba0b1ff3ee +size 1194519 diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log b/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..92b30f7bbf90ddb181aeadc8beb14159ebc5cfe7 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log @@ -0,0 +1,5840 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1734490373.720178]: resetting environment +[INFO] [1734490373.726051]: cleared current plan +[INFO] [1734490373.727315]: moving to home +[INFO] [1734490374.729507]: reached home +[INFO] [1734490374.730649]: reset action history +[INFO] [1734490377.438964]: environment reset +Starting episode 0 +[INFO] [1734490377.441482]: resetting environment +[INFO] [1734490377.444275]: cleared current plan +[INFO] [1734490377.445263]: moving to home +[INFO] [1734490378.447782]: reached home +[INFO] [1734490378.449689]: reset action history +[INFO] [1734490381.158077]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734490399.640330]: resetting environment +[INFO] [1734490399.645467]: cleared current plan +[INFO] [1734490399.646512]: moving to home +[INFO] [1734490401.649270]: reached home +[INFO] [1734490401.650655]: reset action history +[INFO] [1734490404.358984]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/episode_rosbags/episode_0_2024-12-17-21-53-24.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-77808.log +started roslaunch server http://158.130.50.37:46075/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [78124] +process[print_text-2]: started with pid [78125] +process[rosbag_record-3]: started with pid [78126] +started bagging! +[INFO] [1734490405.133579]: received mask_obs with header seq: 1645 +stamp: + secs: 1734490405 + nsecs: 137640476 +frame_id: '' and grasped_object_mask header seq: 125644 +stamp: + secs: 1734490404 + nsecs: 912520170 +frame_id: "camera_color_optical_frame" +[INFO] [1734490405.318614]: received mask_obs with header seq: 1646 +stamp: + secs: 1734490405 + nsecs: 323003053 +frame_id: '' and grasped_object_mask header seq: 125650 +stamp: + secs: 1734490405 + nsecs: 112359047 +frame_id: "camera_color_optical_frame" +[INFO] [1734490405.715046]: received mask_obs with header seq: 1647 +stamp: + secs: 1734490405 + nsecs: 719377517 +frame_id: '' and grasped_object_mask header seq: 125662 +stamp: + secs: 1734490405 + nsecs: 512037039 +frame_id: "camera_color_optical_frame" +[INFO] [1734490405.889254]: received mask_obs with header seq: 1648 +stamp: + secs: 1734490405 + nsecs: 893517971 +frame_id: '' and grasped_object_mask header seq: 125667 +stamp: + secs: 1734490405 + nsecs: 678572416 +frame_id: "camera_color_optical_frame" +[INFO] [1734490406.071535]: received mask_obs with header seq: 1649 +stamp: + secs: 1734490406 + nsecs: 75845718 +frame_id: '' and grasped_object_mask header seq: 125672 +stamp: + secs: 1734490405 + nsecs: 845108271 +frame_id: "camera_color_optical_frame" +[INFO] [1734490406.261055]: received mask_obs with header seq: 1650 +stamp: + secs: 1734490406 + nsecs: 263572216 +frame_id: '' and grasped_object_mask header seq: 125678 +stamp: + secs: 1734490406 + nsecs: 44952869 +frame_id: "camera_color_optical_frame" +[INFO] [1734490406.455169]: received mask_obs with header seq: 1651 +stamp: + secs: 1734490406 + nsecs: 459569692 +frame_id: '' and grasped_object_mask header seq: 125683 +stamp: + secs: 1734490406 + nsecs: 211489201 +frame_id: "camera_color_optical_frame" +[INFO] [1734490406.669947]: received mask_obs with header seq: 1652 +stamp: + secs: 1734490406 + nsecs: 674370527 +frame_id: '' and grasped_object_mask header seq: 125689 +stamp: + secs: 1734490406 + nsecs: 411334038 +frame_id: "camera_color_optical_frame" +[INFO] [1734490406.866041]: received mask_obs with header seq: 1653 +stamp: + secs: 1734490406 + nsecs: 870386123 +frame_id: '' and grasped_object_mask header seq: 125696 +stamp: + secs: 1734490406 + nsecs: 644483805 +frame_id: "camera_color_optical_frame" +[INFO] [1734490407.053365]: received mask_obs with header seq: 1654 +stamp: + secs: 1734490407 + nsecs: 57803153 +frame_id: '' and grasped_object_mask header seq: 125701 +stamp: + secs: 1734490406 + nsecs: 811015606 +frame_id: "camera_color_optical_frame" +[INFO] [1734490407.248672]: received mask_obs with header seq: 1655 +stamp: + secs: 1734490407 + nsecs: 253000736 +frame_id: '' and grasped_object_mask header seq: 125707 +stamp: + secs: 1734490407 + nsecs: 10853052 +frame_id: "camera_color_optical_frame" +[INFO] [1734490407.444390]: received mask_obs with header seq: 1656 +stamp: + secs: 1734490407 + nsecs: 448682069 +frame_id: '' and grasped_object_mask header seq: 125713 +stamp: + secs: 1734490407 + nsecs: 210690498 +frame_id: "camera_color_optical_frame" +[INFO] [1734490407.640426]: received mask_obs with header seq: 1657 +stamp: + secs: 1734490407 + nsecs: 644783020 +frame_id: '' and grasped_object_mask header seq: 125719 +stamp: + secs: 1734490407 + nsecs: 410527706 +frame_id: "camera_color_optical_frame" +[INFO] [1734490407.836191]: received mask_obs with header seq: 1658 +stamp: + secs: 1734490407 + nsecs: 840383529 +frame_id: '' and grasped_object_mask header seq: 125725 +stamp: + secs: 1734490407 + nsecs: 610365152 +frame_id: "camera_color_optical_frame" +[INFO] [1734490408.036794]: received mask_obs with header seq: 1659 +stamp: + secs: 1734490408 + nsecs: 40878534 +frame_id: '' and grasped_object_mask header seq: 125731 +stamp: + secs: 1734490407 + nsecs: 812483311 +frame_id: "camera_color_optical_frame" +[INFO] [1734490408.234161]: received mask_obs with header seq: 1660 +stamp: + secs: 1734490408 + nsecs: 238093614 +frame_id: '' and grasped_object_mask header seq: 125737 +stamp: + secs: 1734490408 + nsecs: 14687300 +frame_id: "camera_color_optical_frame" +[INFO] [1734490408.434242]: received mask_obs with header seq: 1661 +stamp: + secs: 1734490408 + nsecs: 438662767 +frame_id: '' and grasped_object_mask header seq: 125743 +stamp: + secs: 1734490408 + nsecs: 216977119 +frame_id: "camera_color_optical_frame" +[INFO] [1734490408.630409]: received mask_obs with header seq: 1662 +stamp: + secs: 1734490408 + nsecs: 634848356 +frame_id: '' and grasped_object_mask header seq: 125749 +stamp: + secs: 1734490408 + nsecs: 419353485 +frame_id: "camera_color_optical_frame" +[INFO] [1734490408.825544]: received mask_obs with header seq: 1663 +stamp: + secs: 1734490408 + nsecs: 828983545 +frame_id: '' and grasped_object_mask header seq: 125754 +stamp: + secs: 1734490408 + nsecs: 588067055 +frame_id: "camera_color_optical_frame" +[INFO] [1734490409.014306]: received mask_obs with header seq: 1664 +stamp: + secs: 1734490409 + nsecs: 18604516 +frame_id: '' and grasped_object_mask header seq: 125760 +stamp: + secs: 1734490408 + nsecs: 788172722 +frame_id: "camera_color_optical_frame" +[INFO] [1734490409.219272]: received mask_obs with header seq: 1665 +stamp: + secs: 1734490409 + nsecs: 223722696 +frame_id: '' and grasped_object_mask header seq: 125766 +stamp: + secs: 1734490408 + nsecs: 987789392 +frame_id: "camera_color_optical_frame" +[INFO] [1734490409.414753]: received mask_obs with header seq: 1666 +stamp: + secs: 1734490409 + nsecs: 419045925 +frame_id: '' and grasped_object_mask header seq: 125772 +stamp: + secs: 1734490409 + nsecs: 187395096 +frame_id: "camera_color_optical_frame" +[INFO] [1734490409.612622]: received mask_obs with header seq: 1667 +stamp: + secs: 1734490409 + nsecs: 616883277 +frame_id: '' and grasped_object_mask header seq: 125778 +stamp: + secs: 1734490409 + nsecs: 386989355 +frame_id: "camera_color_optical_frame" +[INFO] [1734490409.814924]: received mask_obs with header seq: 1668 +stamp: + secs: 1734490409 + nsecs: 819295167 +frame_id: '' and grasped_object_mask header seq: 125784 +stamp: + secs: 1734490409 + nsecs: 586572409 +frame_id: "camera_color_optical_frame" +[INFO] [1734490410.011932]: received mask_obs with header seq: 1669 +stamp: + secs: 1734490410 + nsecs: 16351461 +frame_id: '' and grasped_object_mask header seq: 125790 +stamp: + secs: 1734490409 + nsecs: 786210537 +frame_id: "camera_color_optical_frame" +[INFO] [1734490410.208408]: received mask_obs with header seq: 1670 +stamp: + secs: 1734490410 + nsecs: 212789058 +frame_id: '' and grasped_object_mask header seq: 125796 +stamp: + secs: 1734490409 + nsecs: 985815048 +frame_id: "camera_color_optical_frame" +[INFO] [1734490410.411993]: received mask_obs with header seq: 1671 +stamp: + secs: 1734490410 + nsecs: 416289806 +frame_id: '' and grasped_object_mask header seq: 125802 +stamp: + secs: 1734490410 + nsecs: 185407639 +frame_id: "camera_color_optical_frame" +[INFO] [1734490410.630913]: received mask_obs with header seq: 1672 +stamp: + secs: 1734490410 + nsecs: 634995698 +frame_id: '' and grasped_object_mask header seq: 125808 +stamp: + secs: 1734490410 + nsecs: 384988785 +frame_id: "camera_color_optical_frame" +[INFO] [1734490410.824062]: received mask_obs with header seq: 1673 +stamp: + secs: 1734490410 + nsecs: 828035593 +frame_id: '' and grasped_object_mask header seq: 125814 +stamp: + secs: 1734490410 + nsecs: 584558010 +frame_id: "camera_color_optical_frame" +[INFO] [1734490411.033083]: received mask_obs with header seq: 1674 +stamp: + secs: 1734490411 + nsecs: 37472963 +frame_id: '' and grasped_object_mask header seq: 125820 +stamp: + secs: 1734490410 + nsecs: 784155846 +frame_id: "camera_color_optical_frame" +[INFO] [1734490411.233333]: received mask_obs with header seq: 1675 +stamp: + secs: 1734490411 + nsecs: 237295150 +frame_id: '' and grasped_object_mask header seq: 125827 +stamp: + secs: 1734490411 + nsecs: 17021894 +frame_id: "camera_color_optical_frame" +[INFO] [1734490411.423419]: received mask_obs with header seq: 1676 +stamp: + secs: 1734490411 + nsecs: 427366256 +frame_id: '' and grasped_object_mask header seq: 125832 +stamp: + secs: 1734490411 + nsecs: 183345556 +frame_id: "camera_color_optical_frame" +[INFO] [1734490411.617098]: received mask_obs with header seq: 1677 +stamp: + secs: 1734490411 + nsecs: 621486902 +frame_id: '' and grasped_object_mask header seq: 125838 +stamp: + secs: 1734490411 + nsecs: 382922173 +frame_id: "camera_color_optical_frame" +[INFO] [1734490411.811562]: received mask_obs with header seq: 1678 +stamp: + secs: 1734490411 + nsecs: 815905332 +frame_id: '' and grasped_object_mask header seq: 125844 +stamp: + secs: 1734490411 + nsecs: 582487106 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.009132]: received mask_obs with header seq: 1679 +stamp: + secs: 1734490412 + nsecs: 13461828 +frame_id: '' and grasped_object_mask header seq: 125850 +stamp: + secs: 1734490411 + nsecs: 782083750 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.202316]: received mask_obs with header seq: 1680 +stamp: + secs: 1734490412 + nsecs: 206453800 +frame_id: '' and grasped_object_mask header seq: 125856 +stamp: + secs: 1734490411 + nsecs: 981684685 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.397284]: received mask_obs with header seq: 1681 +stamp: + secs: 1734490412 + nsecs: 401316642 +frame_id: '' and grasped_object_mask header seq: 125862 +stamp: + secs: 1734490412 + nsecs: 181274414 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.597338]: received mask_obs with header seq: 1682 +stamp: + secs: 1734490412 + nsecs: 601709127 +frame_id: '' and grasped_object_mask header seq: 125868 +stamp: + secs: 1734490412 + nsecs: 380851507 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.795180]: received mask_obs with header seq: 1683 +stamp: + secs: 1734490412 + nsecs: 799362659 +frame_id: '' and grasped_object_mask header seq: 125874 +stamp: + secs: 1734490412 + nsecs: 580416679 +frame_id: "camera_color_optical_frame" +[INFO] [1734490412.994490]: received mask_obs with header seq: 1684 +stamp: + secs: 1734490412 + nsecs: 998844623 +frame_id: '' and grasped_object_mask header seq: 125880 +stamp: + secs: 1734490412 + nsecs: 780016661 +frame_id: "camera_color_optical_frame" +[INFO] [1734490413.185992]: received mask_obs with header seq: 1685 +stamp: + secs: 1734490413 + nsecs: 189961433 +frame_id: '' and grasped_object_mask header seq: 125885 +stamp: + secs: 1734490412 + nsecs: 946356058 +frame_id: "camera_color_optical_frame" +[INFO] [1734490413.378553]: received mask_obs with header seq: 1686 +stamp: + secs: 1734490413 + nsecs: 382924556 +frame_id: '' and grasped_object_mask header seq: 125891 +stamp: + secs: 1734490413 + nsecs: 145951748 +frame_id: "camera_color_optical_frame" +[INFO] [1734490413.572601]: received mask_obs with header seq: 1687 +stamp: + secs: 1734490413 + nsecs: 576939582 +frame_id: '' and grasped_object_mask header seq: 125897 +stamp: + secs: 1734490413 + nsecs: 345536232 +frame_id: "camera_color_optical_frame" +[INFO] [1734490413.767553]: received mask_obs with header seq: 1688 +stamp: + secs: 1734490413 + nsecs: 771808862 +frame_id: '' and grasped_object_mask header seq: 125903 +stamp: + secs: 1734490413 + nsecs: 545108795 +frame_id: "camera_color_optical_frame" +[INFO] [1734490413.965500]: received mask_obs with header seq: 1689 +stamp: + secs: 1734490413 + nsecs: 969848632 +frame_id: '' and grasped_object_mask header seq: 125909 +stamp: + secs: 1734490413 + nsecs: 744701385 +frame_id: "camera_color_optical_frame" +[INFO] [1734490414.164852]: received mask_obs with header seq: 1690 +stamp: + secs: 1734490414 + nsecs: 169212818 +frame_id: '' and grasped_object_mask header seq: 125915 +stamp: + secs: 1734490413 + nsecs: 944306850 +frame_id: "camera_color_optical_frame" +[INFO] [1734490414.362523]: received mask_obs with header seq: 1691 +stamp: + secs: 1734490414 + nsecs: 366117954 +frame_id: '' and grasped_object_mask header seq: 125921 +stamp: + secs: 1734490414 + nsecs: 143901587 +frame_id: "camera_color_optical_frame" +[INFO] [1734490414.559566]: received mask_obs with header seq: 1692 +stamp: + secs: 1734490414 + nsecs: 563719987 +frame_id: '' and grasped_object_mask header seq: 125927 +stamp: + secs: 1734490414 + nsecs: 343483686 +frame_id: "camera_color_optical_frame" +[INFO] [1734490414.763784]: received mask_obs with header seq: 1693 +stamp: + secs: 1734490414 + nsecs: 768093109 +frame_id: '' and grasped_object_mask header seq: 125933 +stamp: + secs: 1734490414 + nsecs: 543053865 +frame_id: "camera_color_optical_frame" +[INFO] [1734490414.963003]: received mask_obs with header seq: 1694 +stamp: + secs: 1734490414 + nsecs: 967274904 +frame_id: '' and grasped_object_mask header seq: 125939 +stamp: + secs: 1734490414 + nsecs: 742648125 +frame_id: "camera_color_optical_frame" +[INFO] [1734490415.160208]: received mask_obs with header seq: 1695 +stamp: + secs: 1734490415 + nsecs: 163365602 +frame_id: '' and grasped_object_mask header seq: 125945 +stamp: + secs: 1734490414 + nsecs: 942258835 +frame_id: "camera_color_optical_frame" +[INFO] [1734490415.355123]: received mask_obs with header seq: 1696 +stamp: + secs: 1734490415 + nsecs: 358259916 +frame_id: '' and grasped_object_mask header seq: 125951 +stamp: + secs: 1734490415 + nsecs: 141858339 +frame_id: "camera_color_optical_frame" +[INFO] [1734490415.549883]: received mask_obs with header seq: 1697 +stamp: + secs: 1734490415 + nsecs: 554102182 +frame_id: '' and grasped_object_mask header seq: 125956 +stamp: + secs: 1734490415 + nsecs: 308182955 +frame_id: "camera_color_optical_frame" +[INFO] [1734490415.758534]: received mask_obs with header seq: 1698 +stamp: + secs: 1734490415 + nsecs: 762657880 +frame_id: '' and grasped_object_mask header seq: 125962 +stamp: + secs: 1734490415 + nsecs: 507761478 +frame_id: "camera_color_optical_frame" +[INFO] [1734490415.952943]: received mask_obs with header seq: 1699 +stamp: + secs: 1734490415 + nsecs: 956313133 +frame_id: '' and grasped_object_mask header seq: 125969 +stamp: + secs: 1734490415 + nsecs: 740620852 +frame_id: "camera_color_optical_frame" +[INFO] [1734490416.148569]: received mask_obs with header seq: 1700 +stamp: + secs: 1734490416 + nsecs: 152718067 +frame_id: '' and grasped_object_mask header seq: 125974 +stamp: + secs: 1734490415 + nsecs: 906964779 +frame_id: "camera_color_optical_frame" +[INFO] [1734490416.341593]: received mask_obs with header seq: 1701 +stamp: + secs: 1734490416 + nsecs: 345726013 +frame_id: '' and grasped_object_mask header seq: 125980 +stamp: + secs: 1734490416 + nsecs: 106566429 +frame_id: "camera_color_optical_frame" +[INFO] [1734490416.534248]: received mask_obs with header seq: 1702 +stamp: + secs: 1734490416 + nsecs: 538510560 +frame_id: '' and grasped_object_mask header seq: 125986 +stamp: + secs: 1734490416 + nsecs: 306156635 +frame_id: "camera_color_optical_frame" +[INFO] [1734490416.733006]: received mask_obs with header seq: 1703 +stamp: + secs: 1734490416 + nsecs: 737178087 +frame_id: '' and grasped_object_mask header seq: 125992 +stamp: + secs: 1734490416 + nsecs: 505735397 +frame_id: "camera_color_optical_frame" +[INFO] [1734490416.931837]: received mask_obs with header seq: 1704 +stamp: + secs: 1734490416 + nsecs: 936213731 +frame_id: '' and grasped_object_mask header seq: 125998 +stamp: + secs: 1734490416 + nsecs: 705325842 +frame_id: "camera_color_optical_frame" +[INFO] [1734490417.129243]: received mask_obs with header seq: 1705 +stamp: + secs: 1734490417 + nsecs: 133613824 +frame_id: '' and grasped_object_mask header seq: 126004 +stamp: + secs: 1734490416 + nsecs: 904942036 +frame_id: "camera_color_optical_frame" +[INFO] [1734490417.330766]: received mask_obs with header seq: 1706 +stamp: + secs: 1734490417 + nsecs: 334751129 +frame_id: '' and grasped_object_mask header seq: 126010 +stamp: + secs: 1734490417 + nsecs: 104547739 +frame_id: "camera_color_optical_frame" +[INFO] [1734490417.529288]: received mask_obs with header seq: 1707 +stamp: + secs: 1734490417 + nsecs: 533425807 +frame_id: '' and grasped_object_mask header seq: 126016 +stamp: + secs: 1734490417 + nsecs: 304141045 +frame_id: "camera_color_optical_frame" +[INFO] [1734490417.729197]: received mask_obs with header seq: 1708 +stamp: + secs: 1734490417 + nsecs: 733483552 +frame_id: '' and grasped_object_mask header seq: 126022 +stamp: + secs: 1734490417 + nsecs: 503723383 +frame_id: "camera_color_optical_frame" +[INFO] [1734490417.930570]: received mask_obs with header seq: 1709 +stamp: + secs: 1734490417 + nsecs: 934796094 +frame_id: '' and grasped_object_mask header seq: 126028 +stamp: + secs: 1734490417 + nsecs: 703315973 +frame_id: "camera_color_optical_frame" +[INFO] [1734490418.130356]: received mask_obs with header seq: 1710 +stamp: + secs: 1734490418 + nsecs: 134227275 +frame_id: '' and grasped_object_mask header seq: 126034 +stamp: + secs: 1734490417 + nsecs: 902932405 +frame_id: "camera_color_optical_frame" +[INFO] [1734490418.330751]: received mask_obs with header seq: 1711 +stamp: + secs: 1734490418 + nsecs: 335093021 +frame_id: '' and grasped_object_mask header seq: 126040 +stamp: + secs: 1734490418 + nsecs: 102537394 +frame_id: "camera_color_optical_frame" +[INFO] [1734490418.531805]: received mask_obs with header seq: 1712 +stamp: + secs: 1734490418 + nsecs: 535878419 +frame_id: '' and grasped_object_mask header seq: 126046 +stamp: + secs: 1734490418 + nsecs: 302132368 +frame_id: "camera_color_optical_frame" +[INFO] [1734490418.730979]: received mask_obs with header seq: 1713 +stamp: + secs: 1734490418 + nsecs: 734946012 +frame_id: '' and grasped_object_mask header seq: 126052 +stamp: + secs: 1734490418 + nsecs: 501714706 +frame_id: "camera_color_optical_frame" +[INFO] [1734490418.930233]: received mask_obs with header seq: 1714 +stamp: + secs: 1734490418 + nsecs: 934482097 +frame_id: '' and grasped_object_mask header seq: 126058 +stamp: + secs: 1734490418 + nsecs: 701307535 +frame_id: "camera_color_optical_frame" +[INFO] [1734490419.129146]: received mask_obs with header seq: 1715 +stamp: + secs: 1734490419 + nsecs: 133255243 +frame_id: '' and grasped_object_mask header seq: 126064 +stamp: + secs: 1734490418 + nsecs: 900924921 +frame_id: "camera_color_optical_frame" +[INFO] [1734490419.330154]: received mask_obs with header seq: 1716 +stamp: + secs: 1734490419 + nsecs: 334506511 +frame_id: '' and grasped_object_mask header seq: 126070 +stamp: + secs: 1734490419 + nsecs: 100530863 +frame_id: "camera_color_optical_frame" +[INFO] [1734490419.529416]: received mask_obs with header seq: 1717 +stamp: + secs: 1734490419 + nsecs: 533754587 +frame_id: '' and grasped_object_mask header seq: 126076 +stamp: + secs: 1734490419 + nsecs: 300125360 +frame_id: "camera_color_optical_frame" +[INFO] [1734490419.729885]: received mask_obs with header seq: 1718 +stamp: + secs: 1734490419 + nsecs: 734076499 +frame_id: '' and grasped_object_mask header seq: 126082 +stamp: + secs: 1734490419 + nsecs: 499709606 +frame_id: "camera_color_optical_frame" +[INFO] [1734490419.929996]: received mask_obs with header seq: 1719 +stamp: + secs: 1734490419 + nsecs: 934252262 +frame_id: '' and grasped_object_mask header seq: 126088 +stamp: + secs: 1734490419 + nsecs: 699301481 +frame_id: "camera_color_optical_frame" +[INFO] [1734490420.129674]: received mask_obs with header seq: 1720 +stamp: + secs: 1734490420 + nsecs: 134036779 +frame_id: '' and grasped_object_mask header seq: 126094 +stamp: + secs: 1734490419 + nsecs: 898914814 +frame_id: "camera_color_optical_frame" +[INFO] [1734490420.330411]: received mask_obs with header seq: 1721 +stamp: + secs: 1734490420 + nsecs: 334778070 +frame_id: '' and grasped_object_mask header seq: 126100 +stamp: + secs: 1734490420 + nsecs: 98516703 +frame_id: "camera_color_optical_frame" +[INFO] [1734490420.531179]: received mask_obs with header seq: 1722 +stamp: + secs: 1734490420 + nsecs: 535131692 +frame_id: '' and grasped_object_mask header seq: 126106 +stamp: + secs: 1734490420 + nsecs: 298106670 +frame_id: "camera_color_optical_frame" +[INFO] [1734490420.731277]: received mask_obs with header seq: 1723 +stamp: + secs: 1734490420 + nsecs: 735621452 +frame_id: '' and grasped_object_mask header seq: 126112 +stamp: + secs: 1734490420 + nsecs: 497685194 +frame_id: "camera_color_optical_frame" +[INFO] [1734490420.930400]: received mask_obs with header seq: 1724 +stamp: + secs: 1734490420 + nsecs: 934736013 +frame_id: '' and grasped_object_mask header seq: 126118 +stamp: + secs: 1734490420 + nsecs: 697272778 +frame_id: "camera_color_optical_frame" +[INFO] [1734490421.130159]: received mask_obs with header seq: 1725 +stamp: + secs: 1734490421 + nsecs: 134524345 +frame_id: '' and grasped_object_mask header seq: 126124 +stamp: + secs: 1734490420 + nsecs: 896882772 +frame_id: "camera_color_optical_frame" +[INFO] [1734490421.331855]: received mask_obs with header seq: 1726 +stamp: + secs: 1734490421 + nsecs: 336046934 +frame_id: '' and grasped_object_mask header seq: 126130 +stamp: + secs: 1734490421 + nsecs: 96480370 +frame_id: "camera_color_optical_frame" +[INFO] [1734490421.537244]: received mask_obs with header seq: 1727 +stamp: + secs: 1734490421 + nsecs: 541592359 +frame_id: '' and grasped_object_mask header seq: 126136 +stamp: + secs: 1734490421 + nsecs: 296066046 +frame_id: "camera_color_optical_frame" +[INFO] [1734490421.735648]: received mask_obs with header seq: 1728 +stamp: + secs: 1734490421 + nsecs: 739858865 +frame_id: '' and grasped_object_mask header seq: 126142 +stamp: + secs: 1734490421 + nsecs: 495640278 +frame_id: "camera_color_optical_frame" +[INFO] [1734490421.929631]: received mask_obs with header seq: 1729 +stamp: + secs: 1734490421 + nsecs: 933944225 +frame_id: '' and grasped_object_mask header seq: 126148 +stamp: + secs: 1734490421 + nsecs: 695224524 +frame_id: "camera_color_optical_frame" +[INFO] [1734490422.140976]: received mask_obs with header seq: 1730 +stamp: + secs: 1734490422 + nsecs: 145223855 +frame_id: '' and grasped_object_mask header seq: 126154 +stamp: + secs: 1734490421 + nsecs: 894833803 +frame_id: "camera_color_optical_frame" +[INFO] [1734490422.338801]: received mask_obs with header seq: 1731 +stamp: + secs: 1734490422 + nsecs: 343151569 +frame_id: '' and grasped_object_mask header seq: 126160 +stamp: + secs: 1734490422 + nsecs: 94432116 +frame_id: "camera_color_optical_frame" +[INFO] [1734490422.547140]: received mask_obs with header seq: 1732 +stamp: + secs: 1734490422 + nsecs: 550071001 +frame_id: '' and grasped_object_mask header seq: 126166 +stamp: + secs: 1734490422 + nsecs: 294017553 +frame_id: "camera_color_optical_frame" +[INFO] [1734490422.738943]: received mask_obs with header seq: 1733 +stamp: + secs: 1734490422 + nsecs: 743086099 +frame_id: '' and grasped_object_mask header seq: 126172 +stamp: + secs: 1734490422 + nsecs: 493591547 +frame_id: "camera_color_optical_frame" +[INFO] [1734490422.934987]: received mask_obs with header seq: 1734 +stamp: + secs: 1734490422 + nsecs: 939305782 +frame_id: '' and grasped_object_mask header seq: 126178 +stamp: + secs: 1734490422 + nsecs: 693175793 +frame_id: "camera_color_optical_frame" +[INFO] [1734490423.130017]: received mask_obs with header seq: 1735 +stamp: + secs: 1734490423 + nsecs: 134113311 +frame_id: '' and grasped_object_mask header seq: 126184 +stamp: + secs: 1734490422 + nsecs: 892785549 +frame_id: "camera_color_optical_frame" +[INFO] [1734490423.331007]: received mask_obs with header seq: 1736 +stamp: + secs: 1734490423 + nsecs: 335229873 +frame_id: '' and grasped_object_mask header seq: 126190 +stamp: + secs: 1734490423 + nsecs: 92384100 +frame_id: "camera_color_optical_frame" +[INFO] [1734490423.528007]: received mask_obs with header seq: 1737 +stamp: + secs: 1734490423 + nsecs: 532337903 +frame_id: '' and grasped_object_mask header seq: 126196 +stamp: + secs: 1734490423 + nsecs: 291971445 +frame_id: "camera_color_optical_frame" +[INFO] [1734490423.729827]: received mask_obs with header seq: 1738 +stamp: + secs: 1734490423 + nsecs: 734209775 +frame_id: '' and grasped_object_mask header seq: 126202 +stamp: + secs: 1734490423 + nsecs: 491546154 +frame_id: "camera_color_optical_frame" +[INFO] [1734490423.940843]: received mask_obs with header seq: 1739 +stamp: + secs: 1734490423 + nsecs: 945054292 +frame_id: '' and grasped_object_mask header seq: 126208 +stamp: + secs: 1734490423 + nsecs: 691587210 +frame_id: "camera_color_optical_frame" +[INFO] [1734490424.133501]: received mask_obs with header seq: 1740 +stamp: + secs: 1734490424 + nsecs: 137856721 +frame_id: '' and grasped_object_mask header seq: 126214 +stamp: + secs: 1734490423 + nsecs: 892508507 +frame_id: "camera_color_optical_frame" +[INFO] [1734490424.329122]: received mask_obs with header seq: 1741 +stamp: + secs: 1734490424 + nsecs: 333485603 +frame_id: '' and grasped_object_mask header seq: 126220 +stamp: + secs: 1734490424 + nsecs: 93516588 +frame_id: "camera_color_optical_frame" +[INFO] [1734490424.528780]: received mask_obs with header seq: 1742 +stamp: + secs: 1734490424 + nsecs: 532914638 +frame_id: '' and grasped_object_mask header seq: 126226 +stamp: + secs: 1734490424 + nsecs: 294611454 +frame_id: "camera_color_optical_frame" +[INFO] [1734490424.728106]: received mask_obs with header seq: 1743 +stamp: + secs: 1734490424 + nsecs: 732480049 +frame_id: '' and grasped_object_mask header seq: 126232 +stamp: + secs: 1734490424 + nsecs: 495794296 +frame_id: "camera_color_optical_frame" +[INFO] [1734490424.928899]: received mask_obs with header seq: 1744 +stamp: + secs: 1734490424 + nsecs: 933155536 +frame_id: '' and grasped_object_mask header seq: 126238 +stamp: + secs: 1734490424 + nsecs: 696563244 +frame_id: "camera_color_optical_frame" +[INFO] [1734490425.128728]: received mask_obs with header seq: 1745 +stamp: + secs: 1734490425 + nsecs: 133108377 +frame_id: '' and grasped_object_mask header seq: 126244 +stamp: + secs: 1734490424 + nsecs: 896402836 +frame_id: "camera_color_optical_frame" +[INFO] [1734490425.330400]: received mask_obs with header seq: 1746 +stamp: + secs: 1734490425 + nsecs: 334378242 +frame_id: '' and grasped_object_mask header seq: 126250 +stamp: + secs: 1734490425 + nsecs: 96242428 +frame_id: "camera_color_optical_frame" +[INFO] [1734490425.529619]: received mask_obs with header seq: 1747 +stamp: + secs: 1734490425 + nsecs: 533796548 +frame_id: '' and grasped_object_mask header seq: 126256 +stamp: + secs: 1734490425 + nsecs: 296082258 +frame_id: "camera_color_optical_frame" +[INFO] [1734490425.728715]: received mask_obs with header seq: 1748 +stamp: + secs: 1734490425 + nsecs: 732555866 +frame_id: '' and grasped_object_mask header seq: 126262 +stamp: + secs: 1734490425 + nsecs: 495922089 +frame_id: "camera_color_optical_frame" +[INFO] [1734490425.943772]: received mask_obs with header seq: 1749 +stamp: + secs: 1734490425 + nsecs: 947026729 +frame_id: '' and grasped_object_mask header seq: 126268 +stamp: + secs: 1734490425 + nsecs: 695761919 +frame_id: "camera_color_optical_frame" +[INFO] [1734490426.133211]: received mask_obs with header seq: 1750 +stamp: + secs: 1734490426 + nsecs: 137365102 +frame_id: '' and grasped_object_mask header seq: 126274 +stamp: + secs: 1734490425 + nsecs: 895598888 +frame_id: "camera_color_optical_frame" +[INFO] [1734490426.329674]: received mask_obs with header seq: 1751 +stamp: + secs: 1734490426 + nsecs: 334010839 +frame_id: '' and grasped_object_mask header seq: 126280 +stamp: + secs: 1734490426 + nsecs: 95435619 +frame_id: "camera_color_optical_frame" +[INFO] [1734490426.530611]: received mask_obs with header seq: 1752 +stamp: + secs: 1734490426 + nsecs: 534654140 +frame_id: '' and grasped_object_mask header seq: 126286 +stamp: + secs: 1734490426 + nsecs: 295272112 +frame_id: "camera_color_optical_frame" +[INFO] [1734490426.728992]: received mask_obs with header seq: 1753 +stamp: + secs: 1734490426 + nsecs: 733244180 +frame_id: '' and grasped_object_mask header seq: 126292 +stamp: + secs: 1734490426 + nsecs: 495108843 +frame_id: "camera_color_optical_frame" +[INFO] [1734490426.928576]: received mask_obs with header seq: 1754 +stamp: + secs: 1734490426 + nsecs: 932901144 +frame_id: '' and grasped_object_mask header seq: 126298 +stamp: + secs: 1734490426 + nsecs: 694946527 +frame_id: "camera_color_optical_frame" +[INFO] [1734490427.128108]: received mask_obs with header seq: 1755 +stamp: + secs: 1734490427 + nsecs: 132277011 +frame_id: '' and grasped_object_mask header seq: 126304 +stamp: + secs: 1734490426 + nsecs: 894786835 +frame_id: "camera_color_optical_frame" +[INFO] [1734490427.329646]: received mask_obs with header seq: 1756 +stamp: + secs: 1734490427 + nsecs: 333978652 +frame_id: '' and grasped_object_mask header seq: 126310 +stamp: + secs: 1734490427 + nsecs: 94628096 +frame_id: "camera_color_optical_frame" +[INFO] [1734490427.529693]: received mask_obs with header seq: 1757 +stamp: + secs: 1734490427 + nsecs: 533935546 +frame_id: '' and grasped_object_mask header seq: 126316 +stamp: + secs: 1734490427 + nsecs: 294468403 +frame_id: "camera_color_optical_frame" +[INFO] [1734490427.729916]: received mask_obs with header seq: 1758 +stamp: + secs: 1734490427 + nsecs: 734201908 +frame_id: '' and grasped_object_mask header seq: 126322 +stamp: + secs: 1734490427 + nsecs: 494308710 +frame_id: "camera_color_optical_frame" +[INFO] [1734490427.929867]: received mask_obs with header seq: 1759 +stamp: + secs: 1734490427 + nsecs: 934169054 +frame_id: '' and grasped_object_mask header seq: 126328 +stamp: + secs: 1734490427 + nsecs: 694147587 +frame_id: "camera_color_optical_frame" +[INFO] [1734490428.129719]: received mask_obs with header seq: 1760 +stamp: + secs: 1734490428 + nsecs: 134037971 +frame_id: '' and grasped_object_mask header seq: 126334 +stamp: + secs: 1734490427 + nsecs: 893983603 +frame_id: "camera_color_optical_frame" +[INFO] [1734490428.329818]: received mask_obs with header seq: 1761 +stamp: + secs: 1734490428 + nsecs: 334218025 +frame_id: '' and grasped_object_mask header seq: 126340 +stamp: + secs: 1734490428 + nsecs: 93819618 +frame_id: "camera_color_optical_frame" +[INFO] [1734490428.546958]: received mask_obs with header seq: 1762 +stamp: + secs: 1734490428 + nsecs: 551317214 +frame_id: '' and grasped_object_mask header seq: 126346 +stamp: + secs: 1734490428 + nsecs: 293656588 +frame_id: "camera_color_optical_frame" +[INFO] [1734490428.751801]: received mask_obs with header seq: 1763 +stamp: + secs: 1734490428 + nsecs: 755934238 +frame_id: '' and grasped_object_mask header seq: 126352 +stamp: + secs: 1734490428 + nsecs: 493492126 +frame_id: "camera_color_optical_frame" +[INFO] [1734490428.955091]: received mask_obs with header seq: 1764 +stamp: + secs: 1734490428 + nsecs: 959496259 +frame_id: '' and grasped_object_mask header seq: 126359 +stamp: + secs: 1734490428 + nsecs: 726632833 +frame_id: "camera_color_optical_frame" +[INFO] [1734490429.153494]: received mask_obs with header seq: 1765 +stamp: + secs: 1734490429 + nsecs: 157430410 +frame_id: '' and grasped_object_mask header seq: 126365 +stamp: + secs: 1734490428 + nsecs: 926468134 +frame_id: "camera_color_optical_frame" +[INFO] [1734490429.350234]: received mask_obs with header seq: 1766 +stamp: + secs: 1734490429 + nsecs: 354577302 +frame_id: '' and grasped_object_mask header seq: 126371 +stamp: + secs: 1734490429 + nsecs: 126302242 +frame_id: "camera_color_optical_frame" +[INFO] [1734490429.548065]: received mask_obs with header seq: 1767 +stamp: + secs: 1734490429 + nsecs: 552374839 +frame_id: '' and grasped_object_mask header seq: 126377 +stamp: + secs: 1734490429 + nsecs: 326136112 +frame_id: "camera_color_optical_frame" +[INFO] [1734490429.744687]: received mask_obs with header seq: 1768 +stamp: + secs: 1734490429 + nsecs: 748975515 +frame_id: '' and grasped_object_mask header seq: 126383 +stamp: + secs: 1734490429 + nsecs: 525970221 +frame_id: "camera_color_optical_frame" +[INFO] [1734490429.932331]: received mask_obs with header seq: 1769 +stamp: + secs: 1734490429 + nsecs: 936636686 +frame_id: '' and grasped_object_mask header seq: 126388 +stamp: + secs: 1734490429 + nsecs: 692499638 +frame_id: "camera_color_optical_frame" +[INFO] [1734490430.129840]: received mask_obs with header seq: 1770 +stamp: + secs: 1734490430 + nsecs: 134196758 +frame_id: '' and grasped_object_mask header seq: 126394 +stamp: + secs: 1734490429 + nsecs: 892336607 +frame_id: "camera_color_optical_frame" +[INFO] [1734490430.330221]: received mask_obs with header seq: 1771 +stamp: + secs: 1734490430 + nsecs: 334565162 +frame_id: '' and grasped_object_mask header seq: 126400 +stamp: + secs: 1734490430 + nsecs: 92173576 +frame_id: "camera_color_optical_frame" +[INFO] [1734490430.529984]: received mask_obs with header seq: 1772 +stamp: + secs: 1734490430 + nsecs: 534355878 +frame_id: '' and grasped_object_mask header seq: 126406 +stamp: + secs: 1734490430 + nsecs: 292010307 +frame_id: "camera_color_optical_frame" +[INFO] [1734490430.730110]: received mask_obs with header seq: 1773 +stamp: + secs: 1734490430 + nsecs: 734456539 +frame_id: '' and grasped_object_mask header seq: 126412 +stamp: + secs: 1734490430 + nsecs: 491846561 +frame_id: "camera_color_optical_frame" +[INFO] [1734490430.930109]: received mask_obs with header seq: 1774 +stamp: + secs: 1734490430 + nsecs: 934496402 +frame_id: '' and grasped_object_mask header seq: 126418 +stamp: + secs: 1734490430 + nsecs: 691686153 +frame_id: "camera_color_optical_frame" +[INFO] [1734490431.129619]: received mask_obs with header seq: 1775 +stamp: + secs: 1734490431 + nsecs: 134004592 +frame_id: '' and grasped_object_mask header seq: 126424 +stamp: + secs: 1734490430 + nsecs: 891527653 +frame_id: "camera_color_optical_frame" +[INFO] [1734490431.329271]: received mask_obs with header seq: 1776 +stamp: + secs: 1734490431 + nsecs: 333658695 +frame_id: '' and grasped_object_mask header seq: 126430 +stamp: + secs: 1734490431 + nsecs: 91369390 +frame_id: "camera_color_optical_frame" +[INFO] [1734490431.375762]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1734490431.530854]: received mask_obs with header seq: 1777 +stamp: + secs: 1734490431 + nsecs: 535179376 +frame_id: '' and grasped_object_mask header seq: 126436 +stamp: + secs: 1734490431 + nsecs: 291211128 +frame_id: "camera_color_optical_frame" +[INFO] [1734490432.113341]: Stored episode 1. +[INFO] [1734490432.116728]: Saving video... +[INFO] [1734490432.307804]: Video saved! +Starting episode 1 +[INFO] [1734490432.320901]: resetting environment +[INFO] [1734490432.330599]: cleared current plan +[INFO] [1734490432.330855]: moving to home +[INFO] [1734490435.933090]: reached home +[INFO] [1734490435.933513]: reset action history +[INFO] [1734490438.645378]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734490441.415476]: resetting environment +[INFO] [1734490441.419313]: cleared current plan +[INFO] [1734490441.419622]: moving to home +[INFO] [1734490442.421261]: reached home +[INFO] [1734490442.421766]: reset action history +[INFO] [1734490445.132770]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/episode_rosbags/episode_1_2024-12-17-21-54-05.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-77808.log +started roslaunch server http://158.130.50.37:38199/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [78232] +process[print_text-5]: started with pid [78233] +process[rosbag_record-6]: started with pid [78257] +started bagging! +[INFO] [1734490445.834217]: received mask_obs with header seq: 1778 +stamp: + secs: 1734490445 + nsecs: 838099241 +frame_id: '' and grasped_object_mask header seq: 126867 +stamp: + secs: 1734490445 + nsecs: 642650127 +frame_id: "camera_color_optical_frame" +[INFO] [1734490446.048488]: received mask_obs with header seq: 1779 +stamp: + secs: 1734490446 + nsecs: 52455902 +frame_id: '' and grasped_object_mask header seq: 126873 +stamp: + secs: 1734490445 + nsecs: 842254162 +frame_id: "camera_color_optical_frame" +[INFO] [1734490446.249714]: received mask_obs with header seq: 1780 +stamp: + secs: 1734490446 + nsecs: 253885507 +frame_id: '' and grasped_object_mask header seq: 126879 +stamp: + secs: 1734490446 + nsecs: 41857004 +frame_id: "camera_color_optical_frame" +[INFO] [1734490446.452033]: received mask_obs with header seq: 1781 +stamp: + secs: 1734490446 + nsecs: 456284523 +frame_id: '' and grasped_object_mask header seq: 126885 +stamp: + secs: 1734490446 + nsecs: 241446733 +frame_id: "camera_color_optical_frame" +[INFO] [1734490446.657553]: received mask_obs with header seq: 1782 +stamp: + secs: 1734490446 + nsecs: 661858320 +frame_id: '' and grasped_object_mask header seq: 126891 +stamp: + secs: 1734490446 + nsecs: 441025019 +frame_id: "camera_color_optical_frame" +[INFO] [1734490446.861327]: received mask_obs with header seq: 1783 +stamp: + secs: 1734490446 + nsecs: 865300893 +frame_id: '' and grasped_object_mask header seq: 126897 +stamp: + secs: 1734490446 + nsecs: 640591145 +frame_id: "camera_color_optical_frame" +[INFO] [1734490447.067721]: received mask_obs with header seq: 1784 +stamp: + secs: 1734490447 + nsecs: 72040081 +frame_id: '' and grasped_object_mask header seq: 126903 +stamp: + secs: 1734490446 + nsecs: 840196133 +frame_id: "camera_color_optical_frame" +[INFO] [1734490447.273957]: received mask_obs with header seq: 1785 +stamp: + secs: 1734490447 + nsecs: 278259277 +frame_id: '' and grasped_object_mask header seq: 126909 +stamp: + secs: 1734490447 + nsecs: 39801598 +frame_id: "camera_color_optical_frame" +[INFO] [1734490447.471803]: received mask_obs with header seq: 1786 +stamp: + secs: 1734490447 + nsecs: 475968360 +frame_id: '' and grasped_object_mask header seq: 126915 +stamp: + secs: 1734490447 + nsecs: 239394903 +frame_id: "camera_color_optical_frame" +[INFO] [1734490447.672645]: received mask_obs with header seq: 1787 +stamp: + secs: 1734490447 + nsecs: 676585674 +frame_id: '' and grasped_object_mask header seq: 126921 +stamp: + secs: 1734490447 + nsecs: 438978195 +frame_id: "camera_color_optical_frame" +[INFO] [1734490447.871782]: received mask_obs with header seq: 1788 +stamp: + secs: 1734490447 + nsecs: 875733852 +frame_id: '' and grasped_object_mask header seq: 126927 +stamp: + secs: 1734490447 + nsecs: 638548613 +frame_id: "camera_color_optical_frame" +[INFO] [1734490448.071414]: received mask_obs with header seq: 1789 +stamp: + secs: 1734490448 + nsecs: 75727939 +frame_id: '' and grasped_object_mask header seq: 126933 +stamp: + secs: 1734490447 + nsecs: 839174271 +frame_id: "camera_color_optical_frame" +[INFO] [1734490448.273052]: received mask_obs with header seq: 1790 +stamp: + secs: 1734490448 + nsecs: 277318477 +frame_id: '' and grasped_object_mask header seq: 126939 +stamp: + secs: 1734490448 + nsecs: 40139675 +frame_id: "camera_color_optical_frame" +[INFO] [1734490448.471313]: received mask_obs with header seq: 1791 +stamp: + secs: 1734490448 + nsecs: 475602626 +frame_id: '' and grasped_object_mask header seq: 126945 +stamp: + secs: 1734490448 + nsecs: 241192818 +frame_id: "camera_color_optical_frame" +[INFO] [1734490448.671515]: received mask_obs with header seq: 1792 +stamp: + secs: 1734490448 + nsecs: 675604581 +frame_id: '' and grasped_object_mask header seq: 126951 +stamp: + secs: 1734490448 + nsecs: 442333460 +frame_id: "camera_color_optical_frame" +[INFO] [1734490448.871323]: received mask_obs with header seq: 1793 +stamp: + secs: 1734490448 + nsecs: 875565767 +frame_id: '' and grasped_object_mask header seq: 126957 +stamp: + secs: 1734490448 + nsecs: 643562794 +frame_id: "camera_color_optical_frame" +[INFO] [1734490449.070897]: received mask_obs with header seq: 1794 +stamp: + secs: 1734490449 + nsecs: 75172424 +frame_id: '' and grasped_object_mask header seq: 126963 +stamp: + secs: 1734490448 + nsecs: 843691111 +frame_id: "camera_color_optical_frame" +[INFO] [1734490449.271179]: received mask_obs with header seq: 1795 +stamp: + secs: 1734490449 + nsecs: 275450944 +frame_id: '' and grasped_object_mask header seq: 126969 +stamp: + secs: 1734490449 + nsecs: 43532133 +frame_id: "camera_color_optical_frame" +[INFO] [1734490449.470461]: received mask_obs with header seq: 1796 +stamp: + secs: 1734490449 + nsecs: 474768400 +frame_id: '' and grasped_object_mask header seq: 126975 +stamp: + secs: 1734490449 + nsecs: 243373632 +frame_id: "camera_color_optical_frame" +[INFO] [1734490449.669767]: received mask_obs with header seq: 1797 +stamp: + secs: 1734490449 + nsecs: 674067258 +frame_id: '' and grasped_object_mask header seq: 126981 +stamp: + secs: 1734490449 + nsecs: 443214893 +frame_id: "camera_color_optical_frame" +[INFO] [1734490449.871388]: received mask_obs with header seq: 1798 +stamp: + secs: 1734490449 + nsecs: 875183582 +frame_id: '' and grasped_object_mask header seq: 126987 +stamp: + secs: 1734490449 + nsecs: 643056393 +frame_id: "camera_color_optical_frame" +[INFO] [1734490450.071993]: received mask_obs with header seq: 1799 +stamp: + secs: 1734490450 + nsecs: 75716972 +frame_id: '' and grasped_object_mask header seq: 126993 +stamp: + secs: 1734490449 + nsecs: 842892647 +frame_id: "camera_color_optical_frame" +[INFO] [1734490450.272345]: received mask_obs with header seq: 1800 +stamp: + secs: 1734490450 + nsecs: 276635646 +frame_id: '' and grasped_object_mask header seq: 126999 +stamp: + secs: 1734490450 + nsecs: 42725801 +frame_id: "camera_color_optical_frame" +[INFO] [1734490450.472120]: received mask_obs with header seq: 1801 +stamp: + secs: 1734490450 + nsecs: 476091861 +frame_id: '' and grasped_object_mask header seq: 127005 +stamp: + secs: 1734490450 + nsecs: 242559433 +frame_id: "camera_color_optical_frame" +[INFO] [1734490450.669061]: received mask_obs with header seq: 1802 +stamp: + secs: 1734490450 + nsecs: 673370838 +frame_id: '' and grasped_object_mask header seq: 127011 +stamp: + secs: 1734490450 + nsecs: 442392588 +frame_id: "camera_color_optical_frame" +[INFO] [1734490450.869043]: received mask_obs with header seq: 1803 +stamp: + secs: 1734490450 + nsecs: 873333692 +frame_id: '' and grasped_object_mask header seq: 127017 +stamp: + secs: 1734490450 + nsecs: 642225504 +frame_id: "camera_color_optical_frame" +[INFO] [1734490451.069819]: received mask_obs with header seq: 1804 +stamp: + secs: 1734490451 + nsecs: 74005842 +frame_id: '' and grasped_object_mask header seq: 127023 +stamp: + secs: 1734490450 + nsecs: 842066765 +frame_id: "camera_color_optical_frame" +[INFO] [1734490451.279019]: received mask_obs with header seq: 1805 +stamp: + secs: 1734490451 + nsecs: 283221006 +frame_id: '' and grasped_object_mask header seq: 127029 +stamp: + secs: 1734490451 + nsecs: 41910887 +frame_id: "camera_color_optical_frame" +[INFO] [1734490451.483031]: received mask_obs with header seq: 1806 +stamp: + secs: 1734490451 + nsecs: 486472368 +frame_id: '' and grasped_object_mask header seq: 127035 +stamp: + secs: 1734490451 + nsecs: 241754055 +frame_id: "camera_color_optical_frame" +[INFO] [1734490451.679363]: received mask_obs with header seq: 1807 +stamp: + secs: 1734490451 + nsecs: 683559417 +frame_id: '' and grasped_object_mask header seq: 127041 +stamp: + secs: 1734490451 + nsecs: 441597700 +frame_id: "camera_color_optical_frame" +[INFO] [1734490451.874747]: received mask_obs with header seq: 1808 +stamp: + secs: 1734490451 + nsecs: 878795862 +frame_id: '' and grasped_object_mask header seq: 127047 +stamp: + secs: 1734490451 + nsecs: 641441107 +frame_id: "camera_color_optical_frame" +[INFO] [1734490452.071677]: received mask_obs with header seq: 1809 +stamp: + secs: 1734490452 + nsecs: 75003147 +frame_id: '' and grasped_object_mask header seq: 127053 +stamp: + secs: 1734490451 + nsecs: 841280222 +frame_id: "camera_color_optical_frame" +[INFO] [1734490452.270042]: received mask_obs with header seq: 1810 +stamp: + secs: 1734490452 + nsecs: 273336648 +frame_id: '' and grasped_object_mask header seq: 127059 +stamp: + secs: 1734490452 + nsecs: 41118383 +frame_id: "camera_color_optical_frame" +[INFO] [1734490452.469263]: received mask_obs with header seq: 1811 +stamp: + secs: 1734490452 + nsecs: 472424507 +frame_id: '' and grasped_object_mask header seq: 127065 +stamp: + secs: 1734490452 + nsecs: 240957260 +frame_id: "camera_color_optical_frame" +[INFO] [1734490452.671557]: received mask_obs with header seq: 1812 +stamp: + secs: 1734490452 + nsecs: 674886226 +frame_id: '' and grasped_object_mask header seq: 127071 +stamp: + secs: 1734490452 + nsecs: 440795422 +frame_id: "camera_color_optical_frame" +[INFO] [1734490452.868874]: received mask_obs with header seq: 1813 +stamp: + secs: 1734490452 + nsecs: 872185945 +frame_id: '' and grasped_object_mask header seq: 127077 +stamp: + secs: 1734490452 + nsecs: 640633345 +frame_id: "camera_color_optical_frame" +[INFO] [1734490453.068132]: received mask_obs with header seq: 1814 +stamp: + secs: 1734490453 + nsecs: 71430683 +frame_id: '' and grasped_object_mask header seq: 127083 +stamp: + secs: 1734490452 + nsecs: 840466022 +frame_id: "camera_color_optical_frame" +[INFO] [1734490453.267576]: received mask_obs with header seq: 1815 +stamp: + secs: 1734490453 + nsecs: 271576881 +frame_id: '' and grasped_object_mask header seq: 127089 +stamp: + secs: 1734490453 + nsecs: 40297508 +frame_id: "camera_color_optical_frame" +[INFO] [1734490453.472385]: received mask_obs with header seq: 1816 +stamp: + secs: 1734490453 + nsecs: 476538419 +frame_id: '' and grasped_object_mask header seq: 127095 +stamp: + secs: 1734490453 + nsecs: 240128756 +frame_id: "camera_color_optical_frame" +[INFO] [1734490453.667449]: received mask_obs with header seq: 1817 +stamp: + secs: 1734490453 + nsecs: 671689271 +frame_id: '' and grasped_object_mask header seq: 127101 +stamp: + secs: 1734490453 + nsecs: 439960480 +frame_id: "camera_color_optical_frame" +[INFO] [1734490453.867383]: received mask_obs with header seq: 1818 +stamp: + secs: 1734490453 + nsecs: 871163606 +frame_id: '' and grasped_object_mask header seq: 127107 +stamp: + secs: 1734490453 + nsecs: 639791012 +frame_id: "camera_color_optical_frame" +[INFO] [1734490454.068816]: received mask_obs with header seq: 1819 +stamp: + secs: 1734490454 + nsecs: 72265625 +frame_id: '' and grasped_object_mask header seq: 127113 +stamp: + secs: 1734490453 + nsecs: 839627266 +frame_id: "camera_color_optical_frame" +[INFO] [1734490454.271958]: received mask_obs with header seq: 1820 +stamp: + secs: 1734490454 + nsecs: 275735855 +frame_id: '' and grasped_object_mask header seq: 127119 +stamp: + secs: 1734490454 + nsecs: 39464712 +frame_id: "camera_color_optical_frame" +[INFO] [1734490454.468496]: received mask_obs with header seq: 1821 +stamp: + secs: 1734490454 + nsecs: 472660303 +frame_id: '' and grasped_object_mask header seq: 127125 +stamp: + secs: 1734490454 + nsecs: 239302158 +frame_id: "camera_color_optical_frame" +[INFO] [1734490454.667050]: received mask_obs with header seq: 1822 +stamp: + secs: 1734490454 + nsecs: 671300411 +frame_id: '' and grasped_object_mask header seq: 127131 +stamp: + secs: 1734490454 + nsecs: 439139843 +frame_id: "camera_color_optical_frame" +[INFO] [1734490454.868014]: received mask_obs with header seq: 1823 +stamp: + secs: 1734490454 + nsecs: 872052669 +frame_id: '' and grasped_object_mask header seq: 127137 +stamp: + secs: 1734490454 + nsecs: 638977528 +frame_id: "camera_color_optical_frame" +[INFO] [1734490455.068077]: received mask_obs with header seq: 1824 +stamp: + secs: 1734490455 + nsecs: 72266340 +frame_id: '' and grasped_object_mask header seq: 127143 +stamp: + secs: 1734490454 + nsecs: 838823318 +frame_id: "camera_color_optical_frame" +[INFO] [1734490455.268609]: received mask_obs with header seq: 1825 +stamp: + secs: 1734490455 + nsecs: 272446393 +frame_id: '' and grasped_object_mask header seq: 127149 +stamp: + secs: 1734490455 + nsecs: 38670301 +frame_id: "camera_color_optical_frame" +[INFO] [1734490455.468080]: received mask_obs with header seq: 1826 +stamp: + secs: 1734490455 + nsecs: 472309350 +frame_id: '' and grasped_object_mask header seq: 127155 +stamp: + secs: 1734490455 + nsecs: 238518000 +frame_id: "camera_color_optical_frame" +[INFO] [1734490455.670023]: received mask_obs with header seq: 1827 +stamp: + secs: 1734490455 + nsecs: 673428535 +frame_id: '' and grasped_object_mask header seq: 127161 +stamp: + secs: 1734490455 + nsecs: 438366175 +frame_id: "camera_color_optical_frame" +[INFO] [1734490455.870520]: received mask_obs with header seq: 1828 +stamp: + secs: 1734490455 + nsecs: 874424457 +frame_id: '' and grasped_object_mask header seq: 127167 +stamp: + secs: 1734490455 + nsecs: 638214588 +frame_id: "camera_color_optical_frame" +[INFO] [1734490456.069953]: received mask_obs with header seq: 1829 +stamp: + secs: 1734490456 + nsecs: 73261737 +frame_id: '' and grasped_object_mask header seq: 127173 +stamp: + secs: 1734490455 + nsecs: 838057280 +frame_id: "camera_color_optical_frame" +[INFO] [1734490456.269420]: received mask_obs with header seq: 1830 +stamp: + secs: 1734490456 + nsecs: 272708177 +frame_id: '' and grasped_object_mask header seq: 127179 +stamp: + secs: 1734490456 + nsecs: 37899733 +frame_id: "camera_color_optical_frame" +[INFO] [1734490456.469129]: received mask_obs with header seq: 1831 +stamp: + secs: 1734490456 + nsecs: 472472667 +frame_id: '' and grasped_object_mask header seq: 127185 +stamp: + secs: 1734490456 + nsecs: 237740993 +frame_id: "camera_color_optical_frame" +[INFO] [1734490456.670323]: received mask_obs with header seq: 1832 +stamp: + secs: 1734490456 + nsecs: 673551321 +frame_id: '' and grasped_object_mask header seq: 127191 +stamp: + secs: 1734490456 + nsecs: 437582254 +frame_id: "camera_color_optical_frame" +[INFO] [1734490456.870745]: received mask_obs with header seq: 1833 +stamp: + secs: 1734490456 + nsecs: 874603033 +frame_id: '' and grasped_object_mask header seq: 127197 +stamp: + secs: 1734490456 + nsecs: 637423515 +frame_id: "camera_color_optical_frame" +[INFO] [1734490457.070552]: received mask_obs with header seq: 1834 +stamp: + secs: 1734490457 + nsecs: 74110746 +frame_id: '' and grasped_object_mask header seq: 127203 +stamp: + secs: 1734490456 + nsecs: 837262392 +frame_id: "camera_color_optical_frame" +[INFO] [1734490457.270221]: received mask_obs with header seq: 1835 +stamp: + secs: 1734490457 + nsecs: 274440526 +frame_id: '' and grasped_object_mask header seq: 127209 +stamp: + secs: 1734490457 + nsecs: 37100554 +frame_id: "camera_color_optical_frame" +[INFO] [1734490457.480085]: received mask_obs with header seq: 1836 +stamp: + secs: 1734490457 + nsecs: 484167575 +frame_id: '' and grasped_object_mask header seq: 127215 +stamp: + secs: 1734490457 + nsecs: 236939907 +frame_id: "camera_color_optical_frame" +[INFO] [1734490457.674742]: received mask_obs with header seq: 1837 +stamp: + secs: 1734490457 + nsecs: 678925514 +frame_id: '' and grasped_object_mask header seq: 127221 +stamp: + secs: 1734490457 + nsecs: 436778307 +frame_id: "camera_color_optical_frame" +[INFO] [1734490457.869857]: received mask_obs with header seq: 1838 +stamp: + secs: 1734490457 + nsecs: 873768806 +frame_id: '' and grasped_object_mask header seq: 127227 +stamp: + secs: 1734490457 + nsecs: 636616945 +frame_id: "camera_color_optical_frame" +[INFO] [1734490458.071220]: received mask_obs with header seq: 1839 +stamp: + secs: 1734490458 + nsecs: 75204849 +frame_id: '' and grasped_object_mask header seq: 127233 +stamp: + secs: 1734490457 + nsecs: 836457491 +frame_id: "camera_color_optical_frame" +[INFO] [1734490458.271761]: received mask_obs with header seq: 1840 +stamp: + secs: 1734490458 + nsecs: 275804758 +frame_id: '' and grasped_object_mask header seq: 127239 +stamp: + secs: 1734490458 + nsecs: 36299229 +frame_id: "camera_color_optical_frame" +[INFO] [1734490458.471727]: received mask_obs with header seq: 1841 +stamp: + secs: 1734490458 + nsecs: 475861549 +frame_id: '' and grasped_object_mask header seq: 127245 +stamp: + secs: 1734490458 + nsecs: 236141205 +frame_id: "camera_color_optical_frame" +[INFO] [1734490458.670974]: received mask_obs with header seq: 1842 +stamp: + secs: 1734490458 + nsecs: 674973964 +frame_id: '' and grasped_object_mask header seq: 127251 +stamp: + secs: 1734490458 + nsecs: 435984135 +frame_id: "camera_color_optical_frame" +[INFO] [1734490458.871105]: received mask_obs with header seq: 1843 +stamp: + secs: 1734490458 + nsecs: 874973297 +frame_id: '' and grasped_object_mask header seq: 127257 +stamp: + secs: 1734490458 + nsecs: 635826111 +frame_id: "camera_color_optical_frame" +[INFO] [1734490459.070731]: received mask_obs with header seq: 1844 +stamp: + secs: 1734490459 + nsecs: 74975490 +frame_id: '' and grasped_object_mask header seq: 127263 +stamp: + secs: 1734490458 + nsecs: 835667133 +frame_id: "camera_color_optical_frame" +[INFO] [1734490459.270938]: received mask_obs with header seq: 1845 +stamp: + secs: 1734490459 + nsecs: 274977684 +frame_id: '' and grasped_object_mask header seq: 127269 +stamp: + secs: 1734490459 + nsecs: 35507917 +frame_id: "camera_color_optical_frame" +[INFO] [1734490459.473919]: received mask_obs with header seq: 1846 +stamp: + secs: 1734490459 + nsecs: 475089550 +frame_id: '' and grasped_object_mask header seq: 127275 +stamp: + secs: 1734490459 + nsecs: 235348463 +frame_id: "camera_color_optical_frame" +[INFO] [1734490459.671141]: received mask_obs with header seq: 1847 +stamp: + secs: 1734490459 + nsecs: 675374746 +frame_id: '' and grasped_object_mask header seq: 127281 +stamp: + secs: 1734490459 + nsecs: 435189486 +frame_id: "camera_color_optical_frame" +[INFO] [1734490459.877964]: received mask_obs with header seq: 1848 +stamp: + secs: 1734490459 + nsecs: 881806850 +frame_id: '' and grasped_object_mask header seq: 127287 +stamp: + secs: 1734490459 + nsecs: 635031462 +frame_id: "camera_color_optical_frame" +[INFO] [1734490460.074310]: received mask_obs with header seq: 1849 +stamp: + secs: 1734490460 + nsecs: 78378915 +frame_id: '' and grasped_object_mask header seq: 127293 +stamp: + secs: 1734490459 + nsecs: 834867716 +frame_id: "camera_color_optical_frame" +[INFO] [1734490460.272932]: received mask_obs with header seq: 1850 +stamp: + secs: 1734490460 + nsecs: 276808977 +frame_id: '' and grasped_object_mask header seq: 127299 +stamp: + secs: 1734490460 + nsecs: 34702301 +frame_id: "camera_color_optical_frame" +[INFO] [1734490460.472942]: received mask_obs with header seq: 1851 +stamp: + secs: 1734490460 + nsecs: 476877927 +frame_id: '' and grasped_object_mask header seq: 127305 +stamp: + secs: 1734490460 + nsecs: 234536886 +frame_id: "camera_color_optical_frame" +[INFO] [1734490460.672410]: received mask_obs with header seq: 1852 +stamp: + secs: 1734490460 + nsecs: 676661729 +frame_id: '' and grasped_object_mask header seq: 127311 +stamp: + secs: 1734490460 + nsecs: 434370995 +frame_id: "camera_color_optical_frame" +[INFO] [1734490460.870920]: received mask_obs with header seq: 1853 +stamp: + secs: 1734490460 + nsecs: 875028133 +frame_id: '' and grasped_object_mask header seq: 127317 +stamp: + secs: 1734490460 + nsecs: 634205103 +frame_id: "camera_color_optical_frame" +[INFO] [1734490461.070822]: received mask_obs with header seq: 1854 +stamp: + secs: 1734490461 + nsecs: 75035810 +frame_id: '' and grasped_object_mask header seq: 127323 +stamp: + secs: 1734490460 + nsecs: 834046125 +frame_id: "camera_color_optical_frame" +[INFO] [1734490461.272599]: received mask_obs with header seq: 1855 +stamp: + secs: 1734490461 + nsecs: 276633501 +frame_id: '' and grasped_object_mask header seq: 127329 +stamp: + secs: 1734490461 + nsecs: 33888102 +frame_id: "camera_color_optical_frame" +[INFO] [1734490461.472086]: received mask_obs with header seq: 1856 +stamp: + secs: 1734490461 + nsecs: 475930213 +frame_id: '' and grasped_object_mask header seq: 127335 +stamp: + secs: 1734490461 + nsecs: 233730555 +frame_id: "camera_color_optical_frame" +[INFO] [1734490461.677877]: received mask_obs with header seq: 1857 +stamp: + secs: 1734490461 + nsecs: 682104349 +frame_id: '' and grasped_object_mask header seq: 127341 +stamp: + secs: 1734490461 + nsecs: 433572769 +frame_id: "camera_color_optical_frame" +[INFO] [1734490461.875720]: received mask_obs with header seq: 1858 +stamp: + secs: 1734490461 + nsecs: 879768133 +frame_id: '' and grasped_object_mask header seq: 127347 +stamp: + secs: 1734490461 + nsecs: 633415461 +frame_id: "camera_color_optical_frame" +[INFO] [1734490462.072947]: received mask_obs with header seq: 1859 +stamp: + secs: 1734490462 + nsecs: 76765298 +frame_id: '' and grasped_object_mask header seq: 127353 +stamp: + secs: 1734490461 + nsecs: 833254099 +frame_id: "camera_color_optical_frame" +[INFO] [1734490462.270960]: received mask_obs with header seq: 1860 +stamp: + secs: 1734490462 + nsecs: 275215387 +frame_id: '' and grasped_object_mask header seq: 127359 +stamp: + secs: 1734490462 + nsecs: 33091068 +frame_id: "camera_color_optical_frame" +[INFO] [1734490462.469548]: received mask_obs with header seq: 1861 +stamp: + secs: 1734490462 + nsecs: 473744153 +frame_id: '' and grasped_object_mask header seq: 127365 +stamp: + secs: 1734490462 + nsecs: 232928991 +frame_id: "camera_color_optical_frame" +[INFO] [1734490462.669543]: received mask_obs with header seq: 1862 +stamp: + secs: 1734490462 + nsecs: 673752069 +frame_id: '' and grasped_object_mask header seq: 127371 +stamp: + secs: 1734490462 + nsecs: 432766199 +frame_id: "camera_color_optical_frame" +[INFO] [1734490462.869862]: received mask_obs with header seq: 1863 +stamp: + secs: 1734490462 + nsecs: 874033689 +frame_id: '' and grasped_object_mask header seq: 127377 +stamp: + secs: 1734490462 + nsecs: 632602692 +frame_id: "camera_color_optical_frame" +[INFO] [1734490463.069455]: received mask_obs with header seq: 1864 +stamp: + secs: 1734490463 + nsecs: 73693037 +frame_id: '' and grasped_object_mask header seq: 127383 +stamp: + secs: 1734490462 + nsecs: 832440615 +frame_id: "camera_color_optical_frame" +[INFO] [1734490463.272080]: received mask_obs with header seq: 1865 +stamp: + secs: 1734490463 + nsecs: 276298284 +frame_id: '' and grasped_object_mask header seq: 127389 +stamp: + secs: 1734490463 + nsecs: 32278776 +frame_id: "camera_color_optical_frame" +[INFO] [1734490463.470243]: received mask_obs with header seq: 1866 +stamp: + secs: 1734490463 + nsecs: 474411249 +frame_id: '' and grasped_object_mask header seq: 127395 +stamp: + secs: 1734490463 + nsecs: 232117176 +frame_id: "camera_color_optical_frame" +[INFO] [1734490463.671138]: received mask_obs with header seq: 1867 +stamp: + secs: 1734490463 + nsecs: 675383329 +frame_id: '' and grasped_object_mask header seq: 127401 +stamp: + secs: 1734490463 + nsecs: 431956291 +frame_id: "camera_color_optical_frame" +[INFO] [1734490463.872528]: received mask_obs with header seq: 1868 +stamp: + secs: 1734490463 + nsecs: 876356124 +frame_id: '' and grasped_object_mask header seq: 127407 +stamp: + secs: 1734490463 + nsecs: 631794691 +frame_id: "camera_color_optical_frame" +[INFO] [1734490464.069882]: received mask_obs with header seq: 1869 +stamp: + secs: 1734490464 + nsecs: 73980331 +frame_id: '' and grasped_object_mask header seq: 127413 +stamp: + secs: 1734490463 + nsecs: 831633091 +frame_id: "camera_color_optical_frame" +[INFO] [1734490464.270591]: received mask_obs with header seq: 1870 +stamp: + secs: 1734490464 + nsecs: 274605035 +frame_id: '' and grasped_object_mask header seq: 127419 +stamp: + secs: 1734490464 + nsecs: 31471491 +frame_id: "camera_color_optical_frame" +[INFO] [1734490464.469851]: received mask_obs with header seq: 1871 +stamp: + secs: 1734490464 + nsecs: 474090337 +frame_id: '' and grasped_object_mask header seq: 127425 +stamp: + secs: 1734490464 + nsecs: 231309652 +frame_id: "camera_color_optical_frame" +[INFO] [1734490464.669841]: received mask_obs with header seq: 1872 +stamp: + secs: 1734490464 + nsecs: 674048662 +frame_id: '' and grasped_object_mask header seq: 127431 +stamp: + secs: 1734490464 + nsecs: 431147575 +frame_id: "camera_color_optical_frame" +[INFO] [1734490464.869970]: received mask_obs with header seq: 1873 +stamp: + secs: 1734490464 + nsecs: 874218940 +frame_id: '' and grasped_object_mask header seq: 127437 +stamp: + secs: 1734490464 + nsecs: 630986691 +frame_id: "camera_color_optical_frame" +[INFO] [1734490465.070235]: received mask_obs with header seq: 1874 +stamp: + secs: 1734490465 + nsecs: 74486494 +frame_id: '' and grasped_object_mask header seq: 127443 +stamp: + secs: 1734490464 + nsecs: 831794739 +frame_id: "camera_color_optical_frame" +[INFO] [1734490465.270084]: received mask_obs with header seq: 1875 +stamp: + secs: 1734490465 + nsecs: 274215221 +frame_id: '' and grasped_object_mask header seq: 127449 +stamp: + secs: 1734490465 + nsecs: 33982992 +frame_id: "camera_color_optical_frame" +[INFO] [1734490465.471272]: received mask_obs with header seq: 1876 +stamp: + secs: 1734490465 + nsecs: 475470304 +frame_id: '' and grasped_object_mask header seq: 127455 +stamp: + secs: 1734490465 + nsecs: 236258507 +frame_id: "camera_color_optical_frame" +[INFO] [1734490465.670494]: received mask_obs with header seq: 1877 +stamp: + secs: 1734490465 + nsecs: 674714803 +frame_id: '' and grasped_object_mask header seq: 127461 +stamp: + secs: 1734490465 + nsecs: 438621521 +frame_id: "camera_color_optical_frame" +[INFO] [1734490465.870777]: received mask_obs with header seq: 1878 +stamp: + secs: 1734490465 + nsecs: 874674797 +frame_id: '' and grasped_object_mask header seq: 127467 +stamp: + secs: 1734490465 + nsecs: 641072512 +frame_id: "camera_color_optical_frame" +[INFO] [1734490466.078023]: received mask_obs with header seq: 1879 +stamp: + secs: 1734490466 + nsecs: 82240581 +frame_id: '' and grasped_object_mask header seq: 127473 +stamp: + secs: 1734490465 + nsecs: 842369795 +frame_id: "camera_color_optical_frame" +[INFO] [1734490466.276315]: received mask_obs with header seq: 1880 +stamp: + secs: 1734490466 + nsecs: 280497312 +frame_id: '' and grasped_object_mask header seq: 127479 +stamp: + secs: 1734490466 + nsecs: 41978121 +frame_id: "camera_color_optical_frame" +[INFO] [1734490466.470100]: received mask_obs with header seq: 1881 +stamp: + secs: 1734490466 + nsecs: 474313020 +frame_id: '' and grasped_object_mask header seq: 127485 +stamp: + secs: 1734490466 + nsecs: 241574526 +frame_id: "camera_color_optical_frame" +[INFO] [1734490466.670137]: received mask_obs with header seq: 1882 +stamp: + secs: 1734490466 + nsecs: 674375057 +frame_id: '' and grasped_object_mask header seq: 127491 +stamp: + secs: 1734490466 + nsecs: 441159248 +frame_id: "camera_color_optical_frame" +[INFO] [1734490466.868601]: received mask_obs with header seq: 1883 +stamp: + secs: 1734490466 + nsecs: 872673034 +frame_id: '' and grasped_object_mask header seq: 127497 +stamp: + secs: 1734490466 + nsecs: 640731573 +frame_id: "camera_color_optical_frame" +[INFO] [1734490467.071210]: received mask_obs with header seq: 1884 +stamp: + secs: 1734490467 + nsecs: 75231552 +frame_id: '' and grasped_object_mask header seq: 127503 +stamp: + secs: 1734490466 + nsecs: 840316296 +frame_id: "camera_color_optical_frame" +[INFO] [1734490467.268425]: received mask_obs with header seq: 1885 +stamp: + secs: 1734490467 + nsecs: 272604703 +frame_id: '' and grasped_object_mask header seq: 127509 +stamp: + secs: 1734490467 + nsecs: 39923429 +frame_id: "camera_color_optical_frame" +[INFO] [1734490467.470470]: received mask_obs with header seq: 1886 +stamp: + secs: 1734490467 + nsecs: 474550008 +frame_id: '' and grasped_object_mask header seq: 127515 +stamp: + secs: 1734490467 + nsecs: 239517689 +frame_id: "camera_color_optical_frame" +[INFO] [1734490467.680190]: received mask_obs with header seq: 1887 +stamp: + secs: 1734490467 + nsecs: 684181690 +frame_id: '' and grasped_object_mask header seq: 127521 +stamp: + secs: 1734490467 + nsecs: 439100266 +frame_id: "camera_color_optical_frame" +[INFO] [1734490467.887625]: received mask_obs with header seq: 1888 +stamp: + secs: 1734490467 + nsecs: 891866445 +frame_id: '' and grasped_object_mask header seq: 127524 +stamp: + secs: 1734490467 + nsecs: 538886786 +frame_id: "camera_color_optical_frame" +[INFO] [1734490468.088812]: received mask_obs with header seq: 1889 +stamp: + secs: 1734490468 + nsecs: 92746257 +frame_id: '' and grasped_object_mask header seq: 127534 +stamp: + secs: 1734490467 + nsecs: 871524096 +frame_id: "camera_color_optical_frame" +[INFO] [1734490468.283164]: received mask_obs with header seq: 1890 +stamp: + secs: 1734490468 + nsecs: 287277936 +frame_id: '' and grasped_object_mask header seq: 127540 +stamp: + secs: 1734490468 + nsecs: 71128607 +frame_id: "camera_color_optical_frame" +[INFO] [1734490468.475873]: received mask_obs with header seq: 1891 +stamp: + secs: 1734490468 + nsecs: 479621171 +frame_id: '' and grasped_object_mask header seq: 127545 +stamp: + secs: 1734490468 + nsecs: 237457275 +frame_id: "camera_color_optical_frame" +[INFO] [1734490468.672522]: received mask_obs with header seq: 1892 +stamp: + secs: 1734490468 + nsecs: 676660060 +frame_id: '' and grasped_object_mask header seq: 127551 +stamp: + secs: 1734490468 + nsecs: 437040329 +frame_id: "camera_color_optical_frame" +[INFO] [1734490468.871145]: received mask_obs with header seq: 1893 +stamp: + secs: 1734490468 + nsecs: 875288724 +frame_id: '' and grasped_object_mask header seq: 127557 +stamp: + secs: 1734490468 + nsecs: 636611462 +frame_id: "camera_color_optical_frame" +[INFO] [1734490469.072371]: received mask_obs with header seq: 1894 +stamp: + secs: 1734490469 + nsecs: 76502323 +frame_id: '' and grasped_object_mask header seq: 127563 +stamp: + secs: 1734490468 + nsecs: 836194277 +frame_id: "camera_color_optical_frame" +[INFO] [1734490469.272545]: received mask_obs with header seq: 1895 +stamp: + secs: 1734490469 + nsecs: 276728391 +frame_id: '' and grasped_object_mask header seq: 127569 +stamp: + secs: 1734490469 + nsecs: 35797834 +frame_id: "camera_color_optical_frame" +[INFO] [1734490469.473970]: received mask_obs with header seq: 1896 +stamp: + secs: 1734490469 + nsecs: 477941274 +frame_id: '' and grasped_object_mask header seq: 127575 +stamp: + secs: 1734490469 + nsecs: 235389471 +frame_id: "camera_color_optical_frame" +[INFO] [1734490469.671559]: received mask_obs with header seq: 1897 +stamp: + secs: 1734490469 + nsecs: 675776004 +frame_id: '' and grasped_object_mask header seq: 127581 +stamp: + secs: 1734490469 + nsecs: 434968948 +frame_id: "camera_color_optical_frame" +[INFO] [1734490469.873058]: received mask_obs with header seq: 1898 +stamp: + secs: 1734490469 + nsecs: 877067327 +frame_id: '' and grasped_object_mask header seq: 127587 +stamp: + secs: 1734490469 + nsecs: 634537458 +frame_id: "camera_color_optical_frame" +[INFO] [1734490470.069708]: received mask_obs with header seq: 1899 +stamp: + secs: 1734490470 + nsecs: 73729038 +frame_id: '' and grasped_object_mask header seq: 127593 +stamp: + secs: 1734490469 + nsecs: 834119558 +frame_id: "camera_color_optical_frame" +[INFO] [1734490470.271043]: received mask_obs with header seq: 1900 +stamp: + secs: 1734490470 + nsecs: 275085210 +frame_id: '' and grasped_object_mask header seq: 127599 +stamp: + secs: 1734490470 + nsecs: 33728361 +frame_id: "camera_color_optical_frame" +[INFO] [1734490470.468483]: received mask_obs with header seq: 1901 +stamp: + secs: 1734490470 + nsecs: 472674608 +frame_id: '' and grasped_object_mask header seq: 127605 +stamp: + secs: 1734490470 + nsecs: 233325005 +frame_id: "camera_color_optical_frame" +[INFO] [1734490470.667566]: received mask_obs with header seq: 1902 +stamp: + secs: 1734490470 + nsecs: 671789646 +frame_id: '' and grasped_object_mask header seq: 127611 +stamp: + secs: 1734490470 + nsecs: 432910204 +frame_id: "camera_color_optical_frame" +[INFO] [1734490470.867528]: received mask_obs with header seq: 1903 +stamp: + secs: 1734490470 + nsecs: 871709108 +frame_id: '' and grasped_object_mask header seq: 127617 +stamp: + secs: 1734490470 + nsecs: 632483721 +frame_id: "camera_color_optical_frame" +[INFO] [1734490471.074594]: received mask_obs with header seq: 1904 +stamp: + secs: 1734490471 + nsecs: 78216552 +frame_id: '' and grasped_object_mask header seq: 127623 +stamp: + secs: 1734490470 + nsecs: 832070827 +frame_id: "camera_color_optical_frame" +[INFO] [1734490471.274403]: received mask_obs with header seq: 1905 +stamp: + secs: 1734490471 + nsecs: 278068065 +frame_id: '' and grasped_object_mask header seq: 127630 +stamp: + secs: 1734490471 + nsecs: 64947128 +frame_id: "camera_color_optical_frame" +[INFO] [1734490471.475110]: received mask_obs with header seq: 1906 +stamp: + secs: 1734490471 + nsecs: 478559732 +frame_id: '' and grasped_object_mask header seq: 127635 +stamp: + secs: 1734490471 + nsecs: 231278896 +frame_id: "camera_color_optical_frame" +[INFO] [1734490471.680107]: received mask_obs with header seq: 1907 +stamp: + secs: 1734490471 + nsecs: 684320211 +frame_id: '' and grasped_object_mask header seq: 127642 +stamp: + secs: 1734490471 + nsecs: 464129210 +frame_id: "camera_color_optical_frame" +[INFO] [1734490471.877127]: received mask_obs with header seq: 1908 +stamp: + secs: 1734490471 + nsecs: 881208896 +frame_id: '' and grasped_object_mask header seq: 127648 +stamp: + secs: 1734490471 + nsecs: 663702726 +frame_id: "camera_color_optical_frame" +[INFO] [1734490472.077276]: received mask_obs with header seq: 1909 +stamp: + secs: 1734490472 + nsecs: 81367492 +frame_id: '' and grasped_object_mask header seq: 127654 +stamp: + secs: 1734490471 + nsecs: 863299131 +frame_id: "camera_color_optical_frame" +[INFO] [1734490472.106982]: Storing episode... +[print_text-5] killing on exit +[pub_text-4] killing on exit +[rosbag_record-6] killing on exit +[WARN] [1734490473.281951]: Plan exhausted +[WARN] [1734490473.324154]: Plan exhausted +[WARN] [1734490473.363160]: Plan exhausted +[WARN] [1734490473.401769]: Plan exhausted +[INFO] [1734490472.885504]: Stored episode 2. +[INFO] [1734490472.885719]: Saving video... +[INFO] [1734490473.381337]: Video saved! +Starting episode 2 +[INFO] [1734490473.413276]: resetting environment +[INFO] [1734490473.426791]: cleared current plan +[INFO] [1734490473.427159]: moving to home +[INFO] [1734490477.031418]: reached home +[INFO] [1734490477.038038]: reset action history +[INFO] [1734490479.820299]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734490486.790448]: resetting environment +[INFO] [1734490486.797283]: cleared current plan +[INFO] [1734490486.797500]: moving to home +[INFO] [1734490487.799044]: reached home +[INFO] [1734490487.799543]: reset action history +[INFO] [1734490490.510131]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/episode_rosbags/episode_2_2024-12-17-21-54-50.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-77808.log +started roslaunch server http://158.130.50.37:40819/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [78369] +process[print_text-8]: started with pid [78370] +process[rosbag_record-9]: started with pid [78371] +started bagging! +[INFO] [1734490491.187055]: received mask_obs with header seq: 1910 +stamp: + secs: 1734490491 + nsecs: 190885305 +frame_id: '' and grasped_object_mask header seq: 128228 +stamp: + secs: 1734490490 + nsecs: 976367950 +frame_id: "camera_color_optical_frame" +[INFO] [1734490491.382802]: received mask_obs with header seq: 1911 +stamp: + secs: 1734490491 + nsecs: 386858701 +frame_id: '' and grasped_object_mask header seq: 128234 +stamp: + secs: 1734490491 + nsecs: 176198244 +frame_id: "camera_color_optical_frame" +[INFO] [1734490491.580722]: received mask_obs with header seq: 1912 +stamp: + secs: 1734490491 + nsecs: 584823369 +frame_id: '' and grasped_object_mask header seq: 128240 +stamp: + secs: 1734490491 + nsecs: 376027107 +frame_id: "camera_color_optical_frame" +[INFO] [1734490491.785532]: received mask_obs with header seq: 1913 +stamp: + secs: 1734490491 + nsecs: 789547681 +frame_id: '' and grasped_object_mask header seq: 128246 +stamp: + secs: 1734490491 + nsecs: 575855732 +frame_id: "camera_color_optical_frame" +[INFO] [1734490491.991914]: received mask_obs with header seq: 1914 +stamp: + secs: 1734490491 + nsecs: 995778083 +frame_id: '' and grasped_object_mask header seq: 128252 +stamp: + secs: 1734490491 + nsecs: 775686979 +frame_id: "camera_color_optical_frame" +[INFO] [1734490492.196623]: received mask_obs with header seq: 1915 +stamp: + secs: 1734490492 + nsecs: 200700759 +frame_id: '' and grasped_object_mask header seq: 128258 +stamp: + secs: 1734490491 + nsecs: 975533962 +frame_id: "camera_color_optical_frame" +[INFO] [1734490492.400127]: received mask_obs with header seq: 1916 +stamp: + secs: 1734490492 + nsecs: 403605222 +frame_id: '' and grasped_object_mask header seq: 128264 +stamp: + secs: 1734490492 + nsecs: 175381184 +frame_id: "camera_color_optical_frame" +[INFO] [1734490492.605662]: received mask_obs with header seq: 1917 +stamp: + secs: 1734490492 + nsecs: 609558105 +frame_id: '' and grasped_object_mask header seq: 128270 +stamp: + secs: 1734490492 + nsecs: 375229836 +frame_id: "camera_color_optical_frame" +[INFO] [1734490492.805602]: received mask_obs with header seq: 1918 +stamp: + secs: 1734490492 + nsecs: 809471845 +frame_id: '' and grasped_object_mask header seq: 128276 +stamp: + secs: 1734490492 + nsecs: 575077295 +frame_id: "camera_color_optical_frame" +[INFO] [1734490493.013255]: received mask_obs with header seq: 1919 +stamp: + secs: 1734490493 + nsecs: 17061471 +frame_id: '' and grasped_object_mask header seq: 128282 +stamp: + secs: 1734490492 + nsecs: 774924278 +frame_id: "camera_color_optical_frame" +[INFO] [1734490493.210103]: received mask_obs with header seq: 1920 +stamp: + secs: 1734490493 + nsecs: 214257717 +frame_id: '' and grasped_object_mask header seq: 128288 +stamp: + secs: 1734490492 + nsecs: 974763632 +frame_id: "camera_color_optical_frame" +[INFO] [1734490493.405026]: received mask_obs with header seq: 1921 +stamp: + secs: 1734490493 + nsecs: 409174203 +frame_id: '' and grasped_object_mask header seq: 128294 +stamp: + secs: 1734490493 + nsecs: 174602747 +frame_id: "camera_color_optical_frame" +[INFO] [1734490493.603207]: received mask_obs with header seq: 1922 +stamp: + secs: 1734490493 + nsecs: 607293128 +frame_id: '' and grasped_object_mask header seq: 128300 +stamp: + secs: 1734490493 + nsecs: 374442101 +frame_id: "camera_color_optical_frame" +[INFO] [1734490493.804755]: received mask_obs with header seq: 1923 +stamp: + secs: 1734490493 + nsecs: 808668375 +frame_id: '' and grasped_object_mask header seq: 128306 +stamp: + secs: 1734490493 + nsecs: 574282169 +frame_id: "camera_color_optical_frame" +[INFO] [1734490494.004637]: received mask_obs with header seq: 1924 +stamp: + secs: 1734490494 + nsecs: 8527040 +frame_id: '' and grasped_object_mask header seq: 128312 +stamp: + secs: 1734490493 + nsecs: 774121046 +frame_id: "camera_color_optical_frame" +[INFO] [1734490494.215866]: received mask_obs with header seq: 1925 +stamp: + secs: 1734490494 + nsecs: 219973087 +frame_id: '' and grasped_object_mask header seq: 128318 +stamp: + secs: 1734490493 + nsecs: 973961592 +frame_id: "camera_color_optical_frame" +[INFO] [1734490494.411710]: received mask_obs with header seq: 1926 +stamp: + secs: 1734490494 + nsecs: 415822029 +frame_id: '' and grasped_object_mask header seq: 128324 +stamp: + secs: 1734490494 + nsecs: 173802137 +frame_id: "camera_color_optical_frame" +[INFO] [1734490494.606865]: received mask_obs with header seq: 1927 +stamp: + secs: 1734490494 + nsecs: 610625267 +frame_id: '' and grasped_object_mask header seq: 128330 +stamp: + secs: 1734490494 + nsecs: 373642921 +frame_id: "camera_color_optical_frame" +[INFO] [1734490494.803769]: received mask_obs with header seq: 1928 +stamp: + secs: 1734490494 + nsecs: 807800054 +frame_id: '' and grasped_object_mask header seq: 128336 +stamp: + secs: 1734490494 + nsecs: 573483706 +frame_id: "camera_color_optical_frame" +[INFO] [1734490495.004482]: received mask_obs with header seq: 1929 +stamp: + secs: 1734490495 + nsecs: 8523464 +frame_id: '' and grasped_object_mask header seq: 128342 +stamp: + secs: 1734490494 + nsecs: 773325443 +frame_id: "camera_color_optical_frame" +[INFO] [1734490495.203059]: received mask_obs with header seq: 1930 +stamp: + secs: 1734490495 + nsecs: 207155466 +frame_id: '' and grasped_object_mask header seq: 128348 +stamp: + secs: 1734490494 + nsecs: 973162174 +frame_id: "camera_color_optical_frame" +[INFO] [1734490495.403113]: received mask_obs with header seq: 1931 +stamp: + secs: 1734490495 + nsecs: 407242059 +frame_id: '' and grasped_object_mask header seq: 128354 +stamp: + secs: 1734490495 + nsecs: 172998428 +frame_id: "camera_color_optical_frame" +[INFO] [1734490495.603422]: received mask_obs with header seq: 1932 +stamp: + secs: 1734490495 + nsecs: 607566595 +frame_id: '' and grasped_object_mask header seq: 128360 +stamp: + secs: 1734490495 + nsecs: 372834921 +frame_id: "camera_color_optical_frame" +[INFO] [1734490495.814841]: received mask_obs with header seq: 1933 +stamp: + secs: 1734490495 + nsecs: 818968534 +frame_id: '' and grasped_object_mask header seq: 128366 +stamp: + secs: 1734490495 + nsecs: 572671175 +frame_id: "camera_color_optical_frame" +[INFO] [1734490496.012686]: received mask_obs with header seq: 1934 +stamp: + secs: 1734490496 + nsecs: 16664981 +frame_id: '' and grasped_object_mask header seq: 128372 +stamp: + secs: 1734490495 + nsecs: 772507429 +frame_id: "camera_color_optical_frame" +[INFO] [1734490496.207682]: received mask_obs with header seq: 1935 +stamp: + secs: 1734490496 + nsecs: 211714744 +frame_id: '' and grasped_object_mask header seq: 128378 +stamp: + secs: 1734490495 + nsecs: 972343922 +frame_id: "camera_color_optical_frame" +[INFO] [1734490496.404116]: received mask_obs with header seq: 1936 +stamp: + secs: 1734490496 + nsecs: 408225774 +frame_id: '' and grasped_object_mask header seq: 128384 +stamp: + secs: 1734490496 + nsecs: 172181129 +frame_id: "camera_color_optical_frame" +[INFO] [1734490496.603145]: received mask_obs with header seq: 1937 +stamp: + secs: 1734490496 + nsecs: 607224225 +frame_id: '' and grasped_object_mask header seq: 128390 +stamp: + secs: 1734490496 + nsecs: 372017384 +frame_id: "camera_color_optical_frame" +[INFO] [1734490496.814228]: received mask_obs with header seq: 1938 +stamp: + secs: 1734490496 + nsecs: 818307399 +frame_id: '' and grasped_object_mask header seq: 128396 +stamp: + secs: 1734490496 + nsecs: 571852922 +frame_id: "camera_color_optical_frame" +[INFO] [1734490497.013304]: received mask_obs with header seq: 1939 +stamp: + secs: 1734490497 + nsecs: 16875505 +frame_id: '' and grasped_object_mask header seq: 128402 +stamp: + secs: 1734490496 + nsecs: 771688461 +frame_id: "camera_color_optical_frame" +[INFO] [1734490497.220305]: received mask_obs with header seq: 1940 +stamp: + secs: 1734490497 + nsecs: 224037647 +frame_id: '' and grasped_object_mask header seq: 128408 +stamp: + secs: 1734490496 + nsecs: 971524000 +frame_id: "camera_color_optical_frame" +[INFO] [1734490497.414457]: received mask_obs with header seq: 1941 +stamp: + secs: 1734490497 + nsecs: 418400764 +frame_id: '' and grasped_object_mask header seq: 128414 +stamp: + secs: 1734490497 + nsecs: 171359539 +frame_id: "camera_color_optical_frame" +[INFO] [1734490497.606914]: received mask_obs with header seq: 1942 +stamp: + secs: 1734490497 + nsecs: 610965967 +frame_id: '' and grasped_object_mask header seq: 128420 +stamp: + secs: 1734490497 + nsecs: 371196032 +frame_id: "camera_color_optical_frame" +[INFO] [1734490497.804742]: received mask_obs with header seq: 1943 +stamp: + secs: 1734490497 + nsecs: 808539628 +frame_id: '' and grasped_object_mask header seq: 128426 +stamp: + secs: 1734490497 + nsecs: 571031332 +frame_id: "camera_color_optical_frame" +[INFO] [1734490498.003719]: received mask_obs with header seq: 1944 +stamp: + secs: 1734490498 + nsecs: 7766246 +frame_id: '' and grasped_object_mask header seq: 128432 +stamp: + secs: 1734490497 + nsecs: 770866394 +frame_id: "camera_color_optical_frame" +[INFO] [1734490498.205665]: received mask_obs with header seq: 1945 +stamp: + secs: 1734490498 + nsecs: 209717512 +frame_id: '' and grasped_object_mask header seq: 128438 +stamp: + secs: 1734490497 + nsecs: 970695734 +frame_id: "camera_color_optical_frame" +[INFO] [1734490498.405585]: received mask_obs with header seq: 1946 +stamp: + secs: 1734490498 + nsecs: 409529447 +frame_id: '' and grasped_object_mask header seq: 128444 +stamp: + secs: 1734490498 + nsecs: 170524359 +frame_id: "camera_color_optical_frame" +[INFO] [1734490498.604798]: received mask_obs with header seq: 1947 +stamp: + secs: 1734490498 + nsecs: 608812332 +frame_id: '' and grasped_object_mask header seq: 128450 +stamp: + secs: 1734490498 + nsecs: 370352745 +frame_id: "camera_color_optical_frame" +[INFO] [1734490498.804589]: received mask_obs with header seq: 1948 +stamp: + secs: 1734490498 + nsecs: 808464288 +frame_id: '' and grasped_object_mask header seq: 128456 +stamp: + secs: 1734490498 + nsecs: 570181608 +frame_id: "camera_color_optical_frame" +[INFO] [1734490499.006954]: received mask_obs with header seq: 1949 +stamp: + secs: 1734490499 + nsecs: 10996580 +frame_id: '' and grasped_object_mask header seq: 128462 +stamp: + secs: 1734490498 + nsecs: 770009041 +frame_id: "camera_color_optical_frame" +[INFO] [1734490499.205396]: received mask_obs with header seq: 1950 +stamp: + secs: 1734490499 + nsecs: 208915948 +frame_id: '' and grasped_object_mask header seq: 128468 +stamp: + secs: 1734490498 + nsecs: 969854116 +frame_id: "camera_color_optical_frame" +[INFO] [1734490499.417093]: received mask_obs with header seq: 1951 +stamp: + secs: 1734490499 + nsecs: 420563220 +frame_id: '' and grasped_object_mask header seq: 128474 +stamp: + secs: 1734490499 + nsecs: 169699192 +frame_id: "camera_color_optical_frame" +[INFO] [1734490499.610224]: received mask_obs with header seq: 1952 +stamp: + secs: 1734490499 + nsecs: 614243984 +frame_id: '' and grasped_object_mask header seq: 128480 +stamp: + secs: 1734490499 + nsecs: 369544744 +frame_id: "camera_color_optical_frame" +[INFO] [1734490499.805450]: received mask_obs with header seq: 1953 +stamp: + secs: 1734490499 + nsecs: 809378147 +frame_id: '' and grasped_object_mask header seq: 128486 +stamp: + secs: 1734490499 + nsecs: 569390297 +frame_id: "camera_color_optical_frame" +[INFO] [1734490500.004844]: received mask_obs with header seq: 1954 +stamp: + secs: 1734490500 + nsecs: 8638858 +frame_id: '' and grasped_object_mask header seq: 128492 +stamp: + secs: 1734490499 + nsecs: 769237518 +frame_id: "camera_color_optical_frame" +[INFO] [1734490500.204351]: received mask_obs with header seq: 1955 +stamp: + secs: 1734490500 + nsecs: 208370208 +frame_id: '' and grasped_object_mask header seq: 128498 +stamp: + secs: 1734490499 + nsecs: 969076395 +frame_id: "camera_color_optical_frame" +[INFO] [1734490500.404490]: received mask_obs with header seq: 1956 +stamp: + secs: 1734490500 + nsecs: 408515214 +frame_id: '' and grasped_object_mask header seq: 128504 +stamp: + secs: 1734490500 + nsecs: 168915272 +frame_id: "camera_color_optical_frame" +[INFO] [1734490500.606545]: received mask_obs with header seq: 1957 +stamp: + secs: 1734490500 + nsecs: 610501527 +frame_id: '' and grasped_object_mask header seq: 128510 +stamp: + secs: 1734490500 + nsecs: 368754387 +frame_id: "camera_color_optical_frame" +[INFO] [1734490500.805766]: received mask_obs with header seq: 1958 +stamp: + secs: 1734490500 + nsecs: 809771537 +frame_id: '' and grasped_object_mask header seq: 128516 +stamp: + secs: 1734490500 + nsecs: 568593740 +frame_id: "camera_color_optical_frame" +[INFO] [1734490501.006077]: received mask_obs with header seq: 1959 +stamp: + secs: 1734490501 + nsecs: 9671926 +frame_id: '' and grasped_object_mask header seq: 128522 +stamp: + secs: 1734490500 + nsecs: 768432856 +frame_id: "camera_color_optical_frame" +[INFO] [1734490501.206239]: received mask_obs with header seq: 1960 +stamp: + secs: 1734490501 + nsecs: 210182666 +frame_id: '' and grasped_object_mask header seq: 128528 +stamp: + secs: 1734490500 + nsecs: 968278408 +frame_id: "camera_color_optical_frame" +[INFO] [1734490501.418299]: received mask_obs with header seq: 1961 +stamp: + secs: 1734490501 + nsecs: 421338081 +frame_id: '' and grasped_object_mask header seq: 128534 +stamp: + secs: 1734490501 + nsecs: 168123007 +frame_id: "camera_color_optical_frame" +[INFO] [1734490501.608875]: received mask_obs with header seq: 1962 +stamp: + secs: 1734490501 + nsecs: 612788200 +frame_id: '' and grasped_object_mask header seq: 128540 +stamp: + secs: 1734490501 + nsecs: 367968082 +frame_id: "camera_color_optical_frame" +[INFO] [1734490501.806776]: received mask_obs with header seq: 1963 +stamp: + secs: 1734490501 + nsecs: 810773134 +frame_id: '' and grasped_object_mask header seq: 128546 +stamp: + secs: 1734490501 + nsecs: 567813396 +frame_id: "camera_color_optical_frame" +[INFO] [1734490502.023225]: received mask_obs with header seq: 1964 +stamp: + secs: 1734490502 + nsecs: 27233600 +frame_id: '' and grasped_object_mask header seq: 128552 +stamp: + secs: 1734490501 + nsecs: 767659187 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 114587212 for nearest: 1734490502082085991 - target: 1734490501967498779 at idx 147 +[INFO] [1734490502.226671]: received mask_obs with header seq: 1965 +stamp: + secs: 1734490502 + nsecs: 230699539 +frame_id: '' and grasped_object_mask header seq: 128558 +stamp: + secs: 1734490501 + nsecs: 967498779 +frame_id: "camera_color_optical_frame" +[INFO] [1734490502.426275]: received mask_obs with header seq: 1966 +stamp: + secs: 1734490502 + nsecs: 430236816 +frame_id: '' and grasped_object_mask header seq: 128565 +stamp: + secs: 1734490502 + nsecs: 200645447 +frame_id: "camera_color_optical_frame" +[INFO] [1734490502.623683]: received mask_obs with header seq: 1967 +stamp: + secs: 1734490502 + nsecs: 627282857 +frame_id: '' and grasped_object_mask header seq: 128571 +stamp: + secs: 1734490502 + nsecs: 400485277 +frame_id: "camera_color_optical_frame" +[INFO] [1734490502.816431]: received mask_obs with header seq: 1968 +stamp: + secs: 1734490502 + nsecs: 818522930 +frame_id: '' and grasped_object_mask header seq: 128576 +stamp: + secs: 1734490502 + nsecs: 567018986 +frame_id: "camera_color_optical_frame" +[INFO] [1734490503.007451]: received mask_obs with header seq: 1969 +stamp: + secs: 1734490503 + nsecs: 11132717 +frame_id: '' and grasped_object_mask header seq: 128582 +stamp: + secs: 1734490502 + nsecs: 766858816 +frame_id: "camera_color_optical_frame" +[INFO] [1734490503.204736]: received mask_obs with header seq: 1970 +stamp: + secs: 1734490503 + nsecs: 208771944 +frame_id: '' and grasped_object_mask header seq: 128588 +stamp: + secs: 1734490502 + nsecs: 966694832 +frame_id: "camera_color_optical_frame" +[INFO] [1734490503.406874]: received mask_obs with header seq: 1971 +stamp: + secs: 1734490503 + nsecs: 410903215 +frame_id: '' and grasped_object_mask header seq: 128594 +stamp: + secs: 1734490503 + nsecs: 166530848 +frame_id: "camera_color_optical_frame" +[INFO] [1734490503.609303]: received mask_obs with header seq: 1972 +stamp: + secs: 1734490503 + nsecs: 613184213 +frame_id: '' and grasped_object_mask header seq: 128600 +stamp: + secs: 1734490503 + nsecs: 366366386 +frame_id: "camera_color_optical_frame" +[INFO] [1734490503.807281]: received mask_obs with header seq: 1973 +stamp: + secs: 1734490503 + nsecs: 811220645 +frame_id: '' and grasped_object_mask header seq: 128606 +stamp: + secs: 1734490503 + nsecs: 566203117 +frame_id: "camera_color_optical_frame" +[INFO] [1734490504.024523]: received mask_obs with header seq: 1974 +stamp: + secs: 1734490504 + nsecs: 28158426 +frame_id: '' and grasped_object_mask header seq: 128612 +stamp: + secs: 1734490503 + nsecs: 766038656 +frame_id: "camera_color_optical_frame" +[INFO] [1734490504.222849]: received mask_obs with header seq: 1975 +stamp: + secs: 1734490504 + nsecs: 226898670 +frame_id: '' and grasped_object_mask header seq: 128619 +stamp: + secs: 1734490503 + nsecs: 999185801 +frame_id: "camera_color_optical_frame" +[INFO] [1734490504.425093]: received mask_obs with header seq: 1976 +stamp: + secs: 1734490504 + nsecs: 428588390 +frame_id: '' and grasped_object_mask header seq: 128625 +stamp: + secs: 1734490504 + nsecs: 199027777 +frame_id: "camera_color_optical_frame" +[INFO] [1734490504.621862]: received mask_obs with header seq: 1977 +stamp: + secs: 1734490504 + nsecs: 625754594 +frame_id: '' and grasped_object_mask header seq: 128631 +stamp: + secs: 1734490504 + nsecs: 398868799 +frame_id: "camera_color_optical_frame" +[INFO] [1734490504.808657]: received mask_obs with header seq: 1978 +stamp: + secs: 1734490504 + nsecs: 812445163 +frame_id: '' and grasped_object_mask header seq: 128636 +stamp: + secs: 1734490504 + nsecs: 565402985 +frame_id: "camera_color_optical_frame" +[INFO] [1734490505.005599]: received mask_obs with header seq: 1979 +stamp: + secs: 1734490505 + nsecs: 9607076 +frame_id: '' and grasped_object_mask header seq: 128642 +stamp: + secs: 1734490504 + nsecs: 765245676 +frame_id: "camera_color_optical_frame" +[INFO] [1734490505.204678]: received mask_obs with header seq: 1980 +stamp: + secs: 1734490505 + nsecs: 208694458 +frame_id: '' and grasped_object_mask header seq: 128648 +stamp: + secs: 1734490504 + nsecs: 965078831 +frame_id: "camera_color_optical_frame" +[INFO] [1734490505.405852]: received mask_obs with header seq: 1981 +stamp: + secs: 1734490505 + nsecs: 409874439 +frame_id: '' and grasped_object_mask header seq: 128654 +stamp: + secs: 1734490505 + nsecs: 164912224 +frame_id: "camera_color_optical_frame" +[INFO] [1734490505.604531]: received mask_obs with header seq: 1982 +stamp: + secs: 1734490505 + nsecs: 608564376 +frame_id: '' and grasped_object_mask header seq: 128660 +stamp: + secs: 1734490505 + nsecs: 364745140 +frame_id: "camera_color_optical_frame" +[INFO] [1734490505.804324]: received mask_obs with header seq: 1983 +stamp: + secs: 1734490505 + nsecs: 808380126 +frame_id: '' and grasped_object_mask header seq: 128666 +stamp: + secs: 1734490505 + nsecs: 564578056 +frame_id: "camera_color_optical_frame" +[INFO] [1734490506.006599]: received mask_obs with header seq: 1984 +stamp: + secs: 1734490506 + nsecs: 10621786 +frame_id: '' and grasped_object_mask header seq: 128672 +stamp: + secs: 1734490505 + nsecs: 764410734 +frame_id: "camera_color_optical_frame" +[INFO] [1734490506.205356]: received mask_obs with header seq: 1985 +stamp: + secs: 1734490506 + nsecs: 209376096 +frame_id: '' and grasped_object_mask header seq: 128678 +stamp: + secs: 1734490505 + nsecs: 964246750 +frame_id: "camera_color_optical_frame" +[INFO] [1734490506.406658]: received mask_obs with header seq: 1986 +stamp: + secs: 1734490506 + nsecs: 410662174 +frame_id: '' and grasped_object_mask header seq: 128684 +stamp: + secs: 1734490506 + nsecs: 164081812 +frame_id: "camera_color_optical_frame" +[INFO] [1734490506.604836]: received mask_obs with header seq: 1987 +stamp: + secs: 1734490506 + nsecs: 608897209 +frame_id: '' and grasped_object_mask header seq: 128690 +stamp: + secs: 1734490506 + nsecs: 363916159 +frame_id: "camera_color_optical_frame" +[INFO] [1734490506.808165]: received mask_obs with header seq: 1988 +stamp: + secs: 1734490506 + nsecs: 811951398 +frame_id: '' and grasped_object_mask header seq: 128696 +stamp: + secs: 1734490506 + nsecs: 563750744 +frame_id: "camera_color_optical_frame" +[INFO] [1734490507.005080]: received mask_obs with header seq: 1989 +stamp: + secs: 1734490507 + nsecs: 9033203 +frame_id: '' and grasped_object_mask header seq: 128702 +stamp: + secs: 1734490506 + nsecs: 763584852 +frame_id: "camera_color_optical_frame" +[INFO] [1734490507.222605]: received mask_obs with header seq: 1990 +stamp: + secs: 1734490507 + nsecs: 226619243 +frame_id: '' and grasped_object_mask header seq: 128708 +stamp: + secs: 1734490506 + nsecs: 963418484 +frame_id: "camera_color_optical_frame" +[INFO] [1734490507.422871]: received mask_obs with header seq: 1991 +stamp: + secs: 1734490507 + nsecs: 426851749 +frame_id: '' and grasped_object_mask header seq: 128715 +stamp: + secs: 1734490507 + nsecs: 196557522 +frame_id: "camera_color_optical_frame" +[INFO] [1734490507.618304]: received mask_obs with header seq: 1992 +stamp: + secs: 1734490507 + nsecs: 622043609 +frame_id: '' and grasped_object_mask header seq: 128721 +stamp: + secs: 1734490507 + nsecs: 396390200 +frame_id: "camera_color_optical_frame" +[INFO] [1734490507.822623]: received mask_obs with header seq: 1993 +stamp: + secs: 1734490507 + nsecs: 826424598 +frame_id: '' and grasped_object_mask header seq: 128726 +stamp: + secs: 1734490507 + nsecs: 562917709 +frame_id: "camera_color_optical_frame" +[INFO] [1734490508.022977]: received mask_obs with header seq: 1994 +stamp: + secs: 1734490508 + nsecs: 27014732 +frame_id: '' and grasped_object_mask header seq: 128733 +stamp: + secs: 1734490507 + nsecs: 796057224 +frame_id: "camera_color_optical_frame" +[INFO] [1734490508.219787]: received mask_obs with header seq: 1995 +stamp: + secs: 1734490508 + nsecs: 223562479 +frame_id: '' and grasped_object_mask header seq: 128739 +stamp: + secs: 1734490507 + nsecs: 995898962 +frame_id: "camera_color_optical_frame" +[INFO] [1734490508.411674]: received mask_obs with header seq: 1996 +stamp: + secs: 1734490508 + nsecs: 415660619 +frame_id: '' and grasped_object_mask header seq: 128745 +stamp: + secs: 1734490508 + nsecs: 195740938 +frame_id: "camera_color_optical_frame" +[INFO] [1734490508.605765]: received mask_obs with header seq: 1997 +stamp: + secs: 1734490508 + nsecs: 609785795 +frame_id: '' and grasped_object_mask header seq: 128750 +stamp: + secs: 1734490508 + nsecs: 362276793 +frame_id: "camera_color_optical_frame" +[INFO] [1734490508.805336]: received mask_obs with header seq: 1998 +stamp: + secs: 1734490508 + nsecs: 809299945 +frame_id: '' and grasped_object_mask header seq: 128756 +stamp: + secs: 1734490508 + nsecs: 562119484 +frame_id: "camera_color_optical_frame" +[INFO] [1734490509.006300]: received mask_obs with header seq: 1999 +stamp: + secs: 1734490509 + nsecs: 10136365 +frame_id: '' and grasped_object_mask header seq: 128762 +stamp: + secs: 1734490508 + nsecs: 761962414 +frame_id: "camera_color_optical_frame" +[INFO] [1734490509.204881]: received mask_obs with header seq: 2000 +stamp: + secs: 1734490509 + nsecs: 208759546 +frame_id: '' and grasped_object_mask header seq: 128768 +stamp: + secs: 1734490508 + nsecs: 961799622 +frame_id: "camera_color_optical_frame" +[INFO] [1734490509.407315]: received mask_obs with header seq: 2001 +stamp: + secs: 1734490509 + nsecs: 411273479 +frame_id: '' and grasped_object_mask header seq: 128774 +stamp: + secs: 1734490509 + nsecs: 161636829 +frame_id: "camera_color_optical_frame" +[INFO] [1734490509.607559]: received mask_obs with header seq: 2002 +stamp: + secs: 1734490509 + nsecs: 611470699 +frame_id: '' and grasped_object_mask header seq: 128780 +stamp: + secs: 1734490509 + nsecs: 361474037 +frame_id: "camera_color_optical_frame" +[INFO] [1734490509.806635]: received mask_obs with header seq: 2003 +stamp: + secs: 1734490509 + nsecs: 810659885 +frame_id: '' and grasped_object_mask header seq: 128786 +stamp: + secs: 1734490509 + nsecs: 561312199 +frame_id: "camera_color_optical_frame" +[INFO] [1734490510.015516]: received mask_obs with header seq: 2004 +stamp: + secs: 1734490510 + nsecs: 18046617 +frame_id: '' and grasped_object_mask header seq: 128792 +stamp: + secs: 1734490509 + nsecs: 761149406 +frame_id: "camera_color_optical_frame" +[INFO] [1734490510.211049]: received mask_obs with header seq: 2005 +stamp: + secs: 1734490510 + nsecs: 213099241 +frame_id: '' and grasped_object_mask header seq: 128798 +stamp: + secs: 1734490509 + nsecs: 960987568 +frame_id: "camera_color_optical_frame" +[INFO] [1734490510.406312]: received mask_obs with header seq: 2006 +stamp: + secs: 1734490510 + nsecs: 410184860 +frame_id: '' and grasped_object_mask header seq: 128804 +stamp: + secs: 1734490510 + nsecs: 160825729 +frame_id: "camera_color_optical_frame" +[INFO] [1734490510.606264]: received mask_obs with header seq: 2007 +stamp: + secs: 1734490510 + nsecs: 610043287 +frame_id: '' and grasped_object_mask header seq: 128810 +stamp: + secs: 1734490510 + nsecs: 360664129 +frame_id: "camera_color_optical_frame" +[INFO] [1734490510.808245]: received mask_obs with header seq: 2008 +stamp: + secs: 1734490510 + nsecs: 812290668 +frame_id: '' and grasped_object_mask header seq: 128816 +stamp: + secs: 1734490510 + nsecs: 560502291 +frame_id: "camera_color_optical_frame" +[INFO] [1734490511.010429]: received mask_obs with header seq: 2009 +stamp: + secs: 1734490511 + nsecs: 12898921 +frame_id: '' and grasped_object_mask header seq: 128822 +stamp: + secs: 1734490510 + nsecs: 760340691 +frame_id: "camera_color_optical_frame" +[INFO] [1734490511.208089]: received mask_obs with header seq: 2010 +stamp: + secs: 1734490511 + nsecs: 211969852 +frame_id: '' and grasped_object_mask header seq: 128828 +stamp: + secs: 1734490510 + nsecs: 960178614 +frame_id: "camera_color_optical_frame" +[INFO] [1734490511.404333]: received mask_obs with header seq: 2011 +stamp: + secs: 1734490511 + nsecs: 408421516 +frame_id: '' and grasped_object_mask header seq: 128834 +stamp: + secs: 1734490511 + nsecs: 160015583 +frame_id: "camera_color_optical_frame" +[INFO] [1734490511.610728]: received mask_obs with header seq: 2012 +stamp: + secs: 1734490511 + nsecs: 614138364 +frame_id: '' and grasped_object_mask header seq: 128841 +stamp: + secs: 1734490511 + nsecs: 393158197 +frame_id: "camera_color_optical_frame" +[INFO] [1734490511.809664]: received mask_obs with header seq: 2013 +stamp: + secs: 1734490511 + nsecs: 812680721 +frame_id: '' and grasped_object_mask header seq: 128846 +stamp: + secs: 1734490511 + nsecs: 559689283 +frame_id: "camera_color_optical_frame" +[INFO] [1734490512.010379]: received mask_obs with header seq: 2014 +stamp: + secs: 1734490512 + nsecs: 12478828 +frame_id: '' and grasped_object_mask header seq: 128852 +stamp: + secs: 1734490511 + nsecs: 759526014 +frame_id: "camera_color_optical_frame" +[INFO] [1734490512.215703]: received mask_obs with header seq: 2015 +stamp: + secs: 1734490512 + nsecs: 219552516 +frame_id: '' and grasped_object_mask header seq: 128858 +stamp: + secs: 1734490511 + nsecs: 959365129 +frame_id: "camera_color_optical_frame" +[INFO] [1734490512.410501]: received mask_obs with header seq: 2016 +stamp: + secs: 1734490512 + nsecs: 412657737 +frame_id: '' and grasped_object_mask header seq: 128865 +stamp: + secs: 1734490512 + nsecs: 192511797 +frame_id: "camera_color_optical_frame" +[INFO] [1734490512.610084]: received mask_obs with header seq: 2017 +stamp: + secs: 1734490512 + nsecs: 613379001 +frame_id: '' and grasped_object_mask header seq: 128870 +stamp: + secs: 1734490512 + nsecs: 359045267 +frame_id: "camera_color_optical_frame" +[INFO] [1734490512.809918]: received mask_obs with header seq: 2018 +stamp: + secs: 1734490512 + nsecs: 813466548 +frame_id: '' and grasped_object_mask header seq: 128876 +stamp: + secs: 1734490512 + nsecs: 558884859 +frame_id: "camera_color_optical_frame" +[INFO] [1734490513.010235]: received mask_obs with header seq: 2019 +stamp: + secs: 1734490513 + nsecs: 13390064 +frame_id: '' and grasped_object_mask header seq: 128882 +stamp: + secs: 1734490512 + nsecs: 758724451 +frame_id: "camera_color_optical_frame" +[INFO] [1734490513.209346]: received mask_obs with header seq: 2020 +stamp: + secs: 1734490513 + nsecs: 212807893 +frame_id: '' and grasped_object_mask header seq: 128888 +stamp: + secs: 1734490512 + nsecs: 958558798 +frame_id: "camera_color_optical_frame" +[INFO] [1734490513.410310]: received mask_obs with header seq: 2021 +stamp: + secs: 1734490513 + nsecs: 414269447 +frame_id: '' and grasped_object_mask header seq: 128895 +stamp: + secs: 1734490513 + nsecs: 191699266 +frame_id: "camera_color_optical_frame" +[INFO] [1734490513.605226]: received mask_obs with header seq: 2022 +stamp: + secs: 1734490513 + nsecs: 609262704 +frame_id: '' and grasped_object_mask header seq: 128900 +stamp: + secs: 1734490513 + nsecs: 358227730 +frame_id: "camera_color_optical_frame" +[INFO] [1734490513.802918]: received mask_obs with header seq: 2023 +stamp: + secs: 1734490513 + nsecs: 806785106 +frame_id: '' and grasped_object_mask header seq: 128906 +stamp: + secs: 1734490513 + nsecs: 558061600 +frame_id: "camera_color_optical_frame" +[INFO] [1734490514.010206]: received mask_obs with header seq: 2024 +stamp: + secs: 1734490514 + nsecs: 12554407 +frame_id: '' and grasped_object_mask header seq: 128912 +stamp: + secs: 1734490513 + nsecs: 757894993 +frame_id: "camera_color_optical_frame" +[INFO] [1734490514.209150]: received mask_obs with header seq: 2025 +stamp: + secs: 1734490514 + nsecs: 212709426 +frame_id: '' and grasped_object_mask header seq: 128918 +stamp: + secs: 1734490513 + nsecs: 957734585 +frame_id: "camera_color_optical_frame" +[INFO] [1734490514.410204]: received mask_obs with header seq: 2026 +stamp: + secs: 1734490514 + nsecs: 413817644 +frame_id: '' and grasped_object_mask header seq: 128925 +stamp: + secs: 1734490514 + nsecs: 190881491 +frame_id: "camera_color_optical_frame" +[INFO] [1734490514.609446]: received mask_obs with header seq: 2027 +stamp: + secs: 1734490514 + nsecs: 612773656 +frame_id: '' and grasped_object_mask header seq: 128930 +stamp: + secs: 1734490514 + nsecs: 357414246 +frame_id: "camera_color_optical_frame" +[INFO] [1734490514.803603]: received mask_obs with header seq: 2028 +stamp: + secs: 1734490514 + nsecs: 807318687 +frame_id: '' and grasped_object_mask header seq: 128936 +stamp: + secs: 1734490514 + nsecs: 557255030 +frame_id: "camera_color_optical_frame" +[INFO] [1734490515.011440]: received mask_obs with header seq: 2029 +stamp: + secs: 1734490515 + nsecs: 15207529 +frame_id: '' and grasped_object_mask header seq: 128942 +stamp: + secs: 1734490514 + nsecs: 757094860 +frame_id: "camera_color_optical_frame" +[INFO] [1734490515.210446]: received mask_obs with header seq: 2030 +stamp: + secs: 1734490515 + nsecs: 214340209 +frame_id: '' and grasped_object_mask header seq: 128948 +stamp: + secs: 1734490514 + nsecs: 956939459 +frame_id: "camera_color_optical_frame" +[INFO] [1734490515.409437]: received mask_obs with header seq: 2031 +stamp: + secs: 1734490515 + nsecs: 412793636 +frame_id: '' and grasped_object_mask header seq: 128954 +stamp: + secs: 1734490515 + nsecs: 156784773 +frame_id: "camera_color_optical_frame" +[INFO] [1734490515.616231]: received mask_obs with header seq: 2032 +stamp: + secs: 1734490515 + nsecs: 620273113 +frame_id: '' and grasped_object_mask header seq: 128960 +stamp: + secs: 1734490515 + nsecs: 356630087 +frame_id: "camera_color_optical_frame" +[INFO] [1734490515.807834]: received mask_obs with header seq: 2033 +stamp: + secs: 1734490515 + nsecs: 811336517 +frame_id: '' and grasped_object_mask header seq: 128967 +stamp: + secs: 1734490515 + nsecs: 589784384 +frame_id: "camera_color_optical_frame" +[INFO] [1734490516.010754]: received mask_obs with header seq: 2034 +stamp: + secs: 1734490516 + nsecs: 14113426 +frame_id: '' and grasped_object_mask header seq: 128973 +stamp: + secs: 1734490515 + nsecs: 789629936 +frame_id: "camera_color_optical_frame" +[INFO] [1734490516.213240]: received mask_obs with header seq: 2035 +stamp: + secs: 1734490516 + nsecs: 217074871 +frame_id: '' and grasped_object_mask header seq: 128979 +stamp: + secs: 1734490515 + nsecs: 989470720 +frame_id: "camera_color_optical_frame" +[INFO] [1734490516.405947]: received mask_obs with header seq: 2036 +stamp: + secs: 1734490516 + nsecs: 409649610 +frame_id: '' and grasped_object_mask header seq: 128985 +stamp: + secs: 1734490516 + nsecs: 189311504 +frame_id: "camera_color_optical_frame" +[INFO] [1734490516.608777]: received mask_obs with header seq: 2037 +stamp: + secs: 1734490516 + nsecs: 612190961 +frame_id: '' and grasped_object_mask header seq: 128990 +stamp: + secs: 1734490516 + nsecs: 355845928 +frame_id: "camera_color_optical_frame" +[INFO] [1734490516.805425]: received mask_obs with header seq: 2038 +stamp: + secs: 1734490516 + nsecs: 809353828 +frame_id: '' and grasped_object_mask header seq: 128997 +stamp: + secs: 1734490516 + nsecs: 588994503 +frame_id: "camera_color_optical_frame" +[INFO] [1734490517.006558]: received mask_obs with header seq: 2039 +stamp: + secs: 1734490517 + nsecs: 10450839 +frame_id: '' and grasped_object_mask header seq: 129003 +stamp: + secs: 1734490516 + nsecs: 788835764 +frame_id: "camera_color_optical_frame" +[INFO] [1734490517.213139]: received mask_obs with header seq: 2040 +stamp: + secs: 1734490517 + nsecs: 217135906 +frame_id: '' and grasped_object_mask header seq: 129009 +stamp: + secs: 1734490516 + nsecs: 988678932 +frame_id: "camera_color_optical_frame" +[INFO] [1734490517.405901]: received mask_obs with header seq: 2041 +stamp: + secs: 1734490517 + nsecs: 409930229 +frame_id: '' and grasped_object_mask header seq: 129014 +stamp: + secs: 1734490517 + nsecs: 155215740 +frame_id: "camera_color_optical_frame" +[INFO] [1734490517.445193]: Storing episode... +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[INFO] [1734490517.610251]: received mask_obs with header seq: 2042 +stamp: + secs: 1734490517 + nsecs: 612665891 +frame_id: '' and grasped_object_mask header seq: 129021 +stamp: + secs: 1734490517 + nsecs: 388366461 +frame_id: "camera_color_optical_frame" +[INFO] [1734490518.240215]: Stored episode 3. +[INFO] [1734490518.242071]: Saving video... +[INFO] [1734490518.568113]: Video saved! +Starting episode 3 +[INFO] [1734490518.615976]: resetting environment +[INFO] [1734490518.632703]: cleared current plan +[INFO] [1734490518.632971]: moving to home +[INFO] [1734490522.235191]: reached home +[INFO] [1734490522.241215]: reset action history +[INFO] [1734490525.013788]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1734490531.589989]: resetting environment +[INFO] [1734490531.598173]: cleared current plan +[INFO] [1734490531.598874]: moving to home +[INFO] [1734490532.601049]: reached home +[INFO] [1734490532.604542]: reset action history +[INFO] [1734490535.339274]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/episode_rosbags/episode_3_2024-12-17-21-55-35.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-77808.log +started roslaunch server http://158.130.50.37:41431/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [78523] +process[print_text-11]: started with pid [78524] +process[rosbag_record-12]: started with pid [78525] +started bagging! +[INFO] [1734490537.236168]: received mask_obs with header seq: 2043 +stamp: + secs: 1734490537 + nsecs: 239578485 +frame_id: '' and grasped_object_mask header seq: 129611 +stamp: + secs: 1734490537 + nsecs: 39097071 +frame_id: "camera_color_optical_frame" +[INFO] [1734490537.441567]: received mask_obs with header seq: 2044 +stamp: + secs: 1734490537 + nsecs: 445377588 +frame_id: '' and grasped_object_mask header seq: 129617 +stamp: + secs: 1734490537 + nsecs: 238945961 +frame_id: "camera_color_optical_frame" +[INFO] [1734490537.642262]: received mask_obs with header seq: 2045 +stamp: + secs: 1734490537 + nsecs: 645674228 +frame_id: '' and grasped_object_mask header seq: 129623 +stamp: + secs: 1734490537 + nsecs: 438796282 +frame_id: "camera_color_optical_frame" +[INFO] [1734490537.845325]: received mask_obs with header seq: 2046 +stamp: + secs: 1734490537 + nsecs: 849234580 +frame_id: '' and grasped_object_mask header seq: 129629 +stamp: + secs: 1734490537 + nsecs: 638646126 +frame_id: "camera_color_optical_frame" +[INFO] [1734490538.048747]: received mask_obs with header seq: 2047 +stamp: + secs: 1734490538 + nsecs: 52699327 +frame_id: '' and grasped_object_mask header seq: 129635 +stamp: + secs: 1734490537 + nsecs: 838493824 +frame_id: "camera_color_optical_frame" +[INFO] [1734490538.263140]: received mask_obs with header seq: 2048 +stamp: + secs: 1734490538 + nsecs: 266618013 +frame_id: '' and grasped_object_mask header seq: 129641 +stamp: + secs: 1734490538 + nsecs: 38338423 +frame_id: "camera_color_optical_frame" +[INFO] [1734490538.463643]: received mask_obs with header seq: 2049 +stamp: + secs: 1734490538 + nsecs: 467351198 +frame_id: '' and grasped_object_mask header seq: 129647 +stamp: + secs: 1734490538 + nsecs: 238183260 +frame_id: "camera_color_optical_frame" +[INFO] [1734490538.665137]: received mask_obs with header seq: 2050 +stamp: + secs: 1734490538 + nsecs: 668955087 +frame_id: '' and grasped_object_mask header seq: 129653 +stamp: + secs: 1734490538 + nsecs: 438028336 +frame_id: "camera_color_optical_frame" +[INFO] [1734490538.866054]: received mask_obs with header seq: 2051 +stamp: + secs: 1734490538 + nsecs: 869620561 +frame_id: '' and grasped_object_mask header seq: 129659 +stamp: + secs: 1734490538 + nsecs: 637874842 +frame_id: "camera_color_optical_frame" +[INFO] [1734490539.063545]: received mask_obs with header seq: 2052 +stamp: + secs: 1734490539 + nsecs: 67507743 +frame_id: '' and grasped_object_mask header seq: 129665 +stamp: + secs: 1734490538 + nsecs: 837719202 +frame_id: "camera_color_optical_frame" +[INFO] [1734490539.263264]: received mask_obs with header seq: 2053 +stamp: + secs: 1734490539 + nsecs: 267220258 +frame_id: '' and grasped_object_mask header seq: 129671 +stamp: + secs: 1734490539 + nsecs: 37559986 +frame_id: "camera_color_optical_frame" +[INFO] [1734490539.464158]: received mask_obs with header seq: 2054 +stamp: + secs: 1734490539 + nsecs: 468114614 +frame_id: '' and grasped_object_mask header seq: 129677 +stamp: + secs: 1734490539 + nsecs: 237400770 +frame_id: "camera_color_optical_frame" +[INFO] [1734490539.664510]: received mask_obs with header seq: 2055 +stamp: + secs: 1734490539 + nsecs: 668349504 +frame_id: '' and grasped_object_mask header seq: 129683 +stamp: + secs: 1734490539 + nsecs: 437242270 +frame_id: "camera_color_optical_frame" +[INFO] [1734490539.864795]: received mask_obs with header seq: 2056 +stamp: + secs: 1734490539 + nsecs: 868598937 +frame_id: '' and grasped_object_mask header seq: 129689 +stamp: + secs: 1734490539 + nsecs: 637083530 +frame_id: "camera_color_optical_frame" +[INFO] [1734490540.064578]: received mask_obs with header seq: 2057 +stamp: + secs: 1734490540 + nsecs: 68272113 +frame_id: '' and grasped_object_mask header seq: 129695 +stamp: + secs: 1734490539 + nsecs: 836926222 +frame_id: "camera_color_optical_frame" +[INFO] [1734490540.264972]: received mask_obs with header seq: 2058 +stamp: + secs: 1734490540 + nsecs: 268823623 +frame_id: '' and grasped_object_mask header seq: 129701 +stamp: + secs: 1734490540 + nsecs: 36768436 +frame_id: "camera_color_optical_frame" +[INFO] [1734490540.464722]: received mask_obs with header seq: 2059 +stamp: + secs: 1734490540 + nsecs: 468653917 +frame_id: '' and grasped_object_mask header seq: 129707 +stamp: + secs: 1734490540 + nsecs: 236611366 +frame_id: "camera_color_optical_frame" +[INFO] [1734490540.661599]: received mask_obs with header seq: 2060 +stamp: + secs: 1734490540 + nsecs: 665554761 +frame_id: '' and grasped_object_mask header seq: 129713 +stamp: + secs: 1734490540 + nsecs: 436454058 +frame_id: "camera_color_optical_frame" +[INFO] [1734490540.863204]: received mask_obs with header seq: 2061 +stamp: + secs: 1734490540 + nsecs: 866699457 +frame_id: '' and grasped_object_mask header seq: 129719 +stamp: + secs: 1734490540 + nsecs: 636297226 +frame_id: "camera_color_optical_frame" +[INFO] [1734490541.063833]: received mask_obs with header seq: 2062 +stamp: + secs: 1734490541 + nsecs: 67541599 +frame_id: '' and grasped_object_mask header seq: 129725 +stamp: + secs: 1734490540 + nsecs: 836139441 +frame_id: "camera_color_optical_frame" +[INFO] [1734490541.263335]: received mask_obs with header seq: 2063 +stamp: + secs: 1734490541 + nsecs: 266797780 +frame_id: '' and grasped_object_mask header seq: 129731 +stamp: + secs: 1734490541 + nsecs: 35980225 +frame_id: "camera_color_optical_frame" +[INFO] [1734490541.460811]: received mask_obs with header seq: 2064 +stamp: + secs: 1734490541 + nsecs: 464364767 +frame_id: '' and grasped_object_mask header seq: 129737 +stamp: + secs: 1734490541 + nsecs: 235820055 +frame_id: "camera_color_optical_frame" +[INFO] [1734490541.660949]: received mask_obs with header seq: 2065 +stamp: + secs: 1734490541 + nsecs: 664178371 +frame_id: '' and grasped_object_mask header seq: 129743 +stamp: + secs: 1734490541 + nsecs: 435659885 +frame_id: "camera_color_optical_frame" +[INFO] [1734490541.861799]: received mask_obs with header seq: 2066 +stamp: + secs: 1734490541 + nsecs: 865452527 +frame_id: '' and grasped_object_mask header seq: 129749 +stamp: + secs: 1734490541 + nsecs: 635499477 +frame_id: "camera_color_optical_frame" +[INFO] [1734490542.067557]: received mask_obs with header seq: 2067 +stamp: + secs: 1734490542 + nsecs: 71349859 +frame_id: '' and grasped_object_mask header seq: 129755 +stamp: + secs: 1734490541 + nsecs: 835339069 +frame_id: "camera_color_optical_frame" +[INFO] [1734490542.264208]: received mask_obs with header seq: 2068 +stamp: + secs: 1734490542 + nsecs: 268089056 +frame_id: '' and grasped_object_mask header seq: 129761 +stamp: + secs: 1734490542 + nsecs: 35179377 +frame_id: "camera_color_optical_frame" +[INFO] [1734490542.460882]: received mask_obs with header seq: 2069 +stamp: + secs: 1734490542 + nsecs: 464839935 +frame_id: '' and grasped_object_mask header seq: 129767 +stamp: + secs: 1734490542 + nsecs: 235019445 +frame_id: "camera_color_optical_frame" +[INFO] [1734490542.660576]: received mask_obs with header seq: 2070 +stamp: + secs: 1734490542 + nsecs: 664376258 +frame_id: '' and grasped_object_mask header seq: 129773 +stamp: + secs: 1734490542 + nsecs: 434860706 +frame_id: "camera_color_optical_frame" +[INFO] [1734490542.859897]: received mask_obs with header seq: 2071 +stamp: + secs: 1734490542 + nsecs: 863704204 +frame_id: '' and grasped_object_mask header seq: 129779 +stamp: + secs: 1734490542 + nsecs: 634701014 +frame_id: "camera_color_optical_frame" +[INFO] [1734490543.059690]: received mask_obs with header seq: 2072 +stamp: + secs: 1734490543 + nsecs: 63343286 +frame_id: '' and grasped_object_mask header seq: 129785 +stamp: + secs: 1734490542 + nsecs: 834540844 +frame_id: "camera_color_optical_frame" +[INFO] [1734490543.259149]: received mask_obs with header seq: 2073 +stamp: + secs: 1734490543 + nsecs: 262867927 +frame_id: '' and grasped_object_mask header seq: 129791 +stamp: + secs: 1734490543 + nsecs: 34380198 +frame_id: "camera_color_optical_frame" +[INFO] [1734490543.459798]: received mask_obs with header seq: 2074 +stamp: + secs: 1734490543 + nsecs: 463075876 +frame_id: '' and grasped_object_mask header seq: 129797 +stamp: + secs: 1734490543 + nsecs: 234219790 +frame_id: "camera_color_optical_frame" +[INFO] [1734490543.660889]: received mask_obs with header seq: 2075 +stamp: + secs: 1734490543 + nsecs: 664680242 +frame_id: '' and grasped_object_mask header seq: 129803 +stamp: + secs: 1734490543 + nsecs: 434059143 +frame_id: "camera_color_optical_frame" +[INFO] [1734490543.859684]: received mask_obs with header seq: 2076 +stamp: + secs: 1734490543 + nsecs: 863516569 +frame_id: '' and grasped_object_mask header seq: 129809 +stamp: + secs: 1734490543 + nsecs: 633899689 +frame_id: "camera_color_optical_frame" +[INFO] [1734490544.059371]: received mask_obs with header seq: 2077 +stamp: + secs: 1734490544 + nsecs: 62875509 +frame_id: '' and grasped_object_mask header seq: 129815 +stamp: + secs: 1734490543 + nsecs: 833738804 +frame_id: "camera_color_optical_frame" +[INFO] [1734490544.259963]: received mask_obs with header seq: 2078 +stamp: + secs: 1734490544 + nsecs: 263833999 +frame_id: '' and grasped_object_mask header seq: 129821 +stamp: + secs: 1734490544 + nsecs: 33576488 +frame_id: "camera_color_optical_frame" +[INFO] [1734490544.458529]: received mask_obs with header seq: 2079 +stamp: + secs: 1734490544 + nsecs: 462399005 +frame_id: '' and grasped_object_mask header seq: 129827 +stamp: + secs: 1734490544 + nsecs: 233414173 +frame_id: "camera_color_optical_frame" +[INFO] [1734490544.658796]: received mask_obs with header seq: 2080 +stamp: + secs: 1734490544 + nsecs: 662684917 +frame_id: '' and grasped_object_mask header seq: 129833 +stamp: + secs: 1734490544 + nsecs: 433251381 +frame_id: "camera_color_optical_frame" +[INFO] [1734490544.866793]: received mask_obs with header seq: 2081 +stamp: + secs: 1734490544 + nsecs: 870211362 +frame_id: '' and grasped_object_mask header seq: 129839 +stamp: + secs: 1734490544 + nsecs: 633088827 +frame_id: "camera_color_optical_frame" +[INFO] [1734490545.059237]: received mask_obs with header seq: 2082 +stamp: + secs: 1734490545 + nsecs: 62743902 +frame_id: '' and grasped_object_mask header seq: 129845 +stamp: + secs: 1734490544 + nsecs: 832926750 +frame_id: "camera_color_optical_frame" +[INFO] [1734490545.260173]: received mask_obs with header seq: 2083 +stamp: + secs: 1734490545 + nsecs: 264072895 +frame_id: '' and grasped_object_mask header seq: 129851 +stamp: + secs: 1734490545 + nsecs: 32763720 +frame_id: "camera_color_optical_frame" +[INFO] [1734490545.457904]: received mask_obs with header seq: 2084 +stamp: + secs: 1734490545 + nsecs: 461187839 +frame_id: '' and grasped_object_mask header seq: 129857 +stamp: + secs: 1734490545 + nsecs: 232600451 +frame_id: "camera_color_optical_frame" +[INFO] [1734490545.657928]: received mask_obs with header seq: 2085 +stamp: + secs: 1734490545 + nsecs: 661788940 +frame_id: '' and grasped_object_mask header seq: 129863 +stamp: + secs: 1734490545 + nsecs: 432436705 +frame_id: "camera_color_optical_frame" +[INFO] [1734490545.858196]: received mask_obs with header seq: 2086 +stamp: + secs: 1734490545 + nsecs: 862030029 +frame_id: '' and grasped_object_mask header seq: 129869 +stamp: + secs: 1734490545 + nsecs: 632273197 +frame_id: "camera_color_optical_frame" +[INFO] [1734490546.058384]: received mask_obs with header seq: 2087 +stamp: + secs: 1734490546 + nsecs: 62168598 +frame_id: '' and grasped_object_mask header seq: 129875 +stamp: + secs: 1734490545 + nsecs: 832109451 +frame_id: "camera_color_optical_frame" +[INFO] [1734490546.258397]: received mask_obs with header seq: 2088 +stamp: + secs: 1734490546 + nsecs: 262230157 +frame_id: '' and grasped_object_mask header seq: 129881 +stamp: + secs: 1734490546 + nsecs: 31945467 +frame_id: "camera_color_optical_frame" +[INFO] [1734490546.458483]: received mask_obs with header seq: 2089 +stamp: + secs: 1734490546 + nsecs: 462065458 +frame_id: '' and grasped_object_mask header seq: 129887 +stamp: + secs: 1734490546 + nsecs: 231780529 +frame_id: "camera_color_optical_frame" +[INFO] [1734490546.658907]: received mask_obs with header seq: 2090 +stamp: + secs: 1734490546 + nsecs: 662636041 +frame_id: '' and grasped_object_mask header seq: 129893 +stamp: + secs: 1734490546 + nsecs: 431615353 +frame_id: "camera_color_optical_frame" +[INFO] [1734490546.860631]: received mask_obs with header seq: 2091 +stamp: + secs: 1734490546 + nsecs: 864416837 +frame_id: '' and grasped_object_mask header seq: 129899 +stamp: + secs: 1734490546 + nsecs: 631449699 +frame_id: "camera_color_optical_frame" +[INFO] [1734490547.058239]: received mask_obs with header seq: 2092 +stamp: + secs: 1734490547 + nsecs: 62121391 +frame_id: '' and grasped_object_mask header seq: 129905 +stamp: + secs: 1734490546 + nsecs: 831285954 +frame_id: "camera_color_optical_frame" +[INFO] [1734490547.258038]: received mask_obs with header seq: 2093 +stamp: + secs: 1734490547 + nsecs: 261396408 +frame_id: '' and grasped_object_mask header seq: 129911 +stamp: + secs: 1734490547 + nsecs: 31130552 +frame_id: "camera_color_optical_frame" +[INFO] [1734490547.466526]: received mask_obs with header seq: 2094 +stamp: + secs: 1734490547 + nsecs: 470378637 +frame_id: '' and grasped_object_mask header seq: 129917 +stamp: + secs: 1734490547 + nsecs: 230976343 +frame_id: "camera_color_optical_frame" +[INFO] [1734490547.671897]: received mask_obs with header seq: 2095 +stamp: + secs: 1734490547 + nsecs: 675487279 +frame_id: '' and grasped_object_mask header seq: 129923 +stamp: + secs: 1734490547 + nsecs: 430821419 +frame_id: "camera_color_optical_frame" +[INFO] [1734490547.865737]: received mask_obs with header seq: 2096 +stamp: + secs: 1734490547 + nsecs: 869640111 +frame_id: '' and grasped_object_mask header seq: 129929 +stamp: + secs: 1734490547 + nsecs: 630667210 +frame_id: "camera_color_optical_frame" +[INFO] [1734490548.060504]: received mask_obs with header seq: 2097 +stamp: + secs: 1734490548 + nsecs: 64072370 +frame_id: '' and grasped_object_mask header seq: 129935 +stamp: + secs: 1734490547 + nsecs: 830510139 +frame_id: "camera_color_optical_frame" +[INFO] [1734490548.257376]: received mask_obs with header seq: 2098 +stamp: + secs: 1734490548 + nsecs: 261232376 +frame_id: '' and grasped_object_mask header seq: 129941 +stamp: + secs: 1734490548 + nsecs: 30342817 +frame_id: "camera_color_optical_frame" +[INFO] [1734490548.460498]: received mask_obs with header seq: 2099 +stamp: + secs: 1734490548 + nsecs: 464346885 +frame_id: '' and grasped_object_mask header seq: 129947 +stamp: + secs: 1734490548 + nsecs: 230174780 +frame_id: "camera_color_optical_frame" +[INFO] [1734490548.655706]: received mask_obs with header seq: 2100 +stamp: + secs: 1734490548 + nsecs: 659397363 +frame_id: '' and grasped_object_mask header seq: 129953 +stamp: + secs: 1734490548 + nsecs: 430007458 +frame_id: "camera_color_optical_frame" +[INFO] [1734490548.855413]: received mask_obs with header seq: 2101 +stamp: + secs: 1734490548 + nsecs: 858759641 +frame_id: '' and grasped_object_mask header seq: 129959 +stamp: + secs: 1734490548 + nsecs: 629838943 +frame_id: "camera_color_optical_frame" +[INFO] [1734490549.055840]: received mask_obs with header seq: 2102 +stamp: + secs: 1734490549 + nsecs: 59606313 +frame_id: '' and grasped_object_mask header seq: 129965 +stamp: + secs: 1734490548 + nsecs: 829670906 +frame_id: "camera_color_optical_frame" +[INFO] [1734490549.257367]: received mask_obs with header seq: 2103 +stamp: + secs: 1734490549 + nsecs: 261062622 +frame_id: '' and grasped_object_mask header seq: 129971 +stamp: + secs: 1734490549 + nsecs: 29507160 +frame_id: "camera_color_optical_frame" +[INFO] [1734490549.454998]: received mask_obs with header seq: 2104 +stamp: + secs: 1734490549 + nsecs: 458739995 +frame_id: '' and grasped_object_mask header seq: 129977 +stamp: + secs: 1734490549 + nsecs: 229342699 +frame_id: "camera_color_optical_frame" +[INFO] [1734490549.655868]: received mask_obs with header seq: 2105 +stamp: + secs: 1734490549 + nsecs: 659698009 +frame_id: '' and grasped_object_mask header seq: 129983 +stamp: + secs: 1734490549 + nsecs: 429178476 +frame_id: "camera_color_optical_frame" +[INFO] [1734490549.855501]: received mask_obs with header seq: 2106 +stamp: + secs: 1734490549 + nsecs: 859339714 +frame_id: '' and grasped_object_mask header seq: 129989 +stamp: + secs: 1734490549 + nsecs: 629014015 +frame_id: "camera_color_optical_frame" +[INFO] [1734490550.058841]: received mask_obs with header seq: 2107 +stamp: + secs: 1734490550 + nsecs: 62626600 +frame_id: '' and grasped_object_mask header seq: 129995 +stamp: + secs: 1734490549 + nsecs: 828850508 +frame_id: "camera_color_optical_frame" +[INFO] [1734490550.255901]: received mask_obs with header seq: 2108 +stamp: + secs: 1734490550 + nsecs: 259537220 +frame_id: '' and grasped_object_mask header seq: 130001 +stamp: + secs: 1734490550 + nsecs: 28688192 +frame_id: "camera_color_optical_frame" +[INFO] [1734490550.455924]: received mask_obs with header seq: 2109 +stamp: + secs: 1734490550 + nsecs: 459740400 +frame_id: '' and grasped_object_mask header seq: 130007 +stamp: + secs: 1734490550 + nsecs: 228525639 +frame_id: "camera_color_optical_frame" +[INFO] [1734490550.655352]: received mask_obs with header seq: 2110 +stamp: + secs: 1734490550 + nsecs: 659004688 +frame_id: '' and grasped_object_mask header seq: 130013 +stamp: + secs: 1734490550 + nsecs: 428363085 +frame_id: "camera_color_optical_frame" +[INFO] [1734490550.855586]: received mask_obs with header seq: 2111 +stamp: + secs: 1734490550 + nsecs: 859425306 +frame_id: '' and grasped_object_mask header seq: 130019 +stamp: + secs: 1734490550 + nsecs: 628200531 +frame_id: "camera_color_optical_frame" +[INFO] [1734490551.054863]: received mask_obs with header seq: 2112 +stamp: + secs: 1734490551 + nsecs: 58302879 +frame_id: '' and grasped_object_mask header seq: 130025 +stamp: + secs: 1734490550 + nsecs: 828037500 +frame_id: "camera_color_optical_frame" +[INFO] [1734490551.257087]: received mask_obs with header seq: 2113 +stamp: + secs: 1734490551 + nsecs: 260793685 +frame_id: '' and grasped_object_mask header seq: 130031 +stamp: + secs: 1734490551 + nsecs: 27874708 +frame_id: "camera_color_optical_frame" +[INFO] [1734490551.455879]: received mask_obs with header seq: 2114 +stamp: + secs: 1734490551 + nsecs: 459614753 +frame_id: '' and grasped_object_mask header seq: 130037 +stamp: + secs: 1734490551 + nsecs: 227710724 +frame_id: "camera_color_optical_frame" +[INFO] [1734490551.662606]: received mask_obs with header seq: 2115 +stamp: + secs: 1734490551 + nsecs: 666151523 +frame_id: '' and grasped_object_mask header seq: 130043 +stamp: + secs: 1734490551 + nsecs: 427546501 +frame_id: "camera_color_optical_frame" +[INFO] [1734490551.857928]: received mask_obs with header seq: 2116 +stamp: + secs: 1734490551 + nsecs: 861805915 +frame_id: '' and grasped_object_mask header seq: 130049 +stamp: + secs: 1734490551 + nsecs: 627381802 +frame_id: "camera_color_optical_frame" +[INFO] [1734490552.058072]: received mask_obs with header seq: 2117 +stamp: + secs: 1734490552 + nsecs: 61933994 +frame_id: '' and grasped_object_mask header seq: 130055 +stamp: + secs: 1734490551 + nsecs: 827217579 +frame_id: "camera_color_optical_frame" +[INFO] [1734490552.254729]: received mask_obs with header seq: 2118 +stamp: + secs: 1734490552 + nsecs: 258485794 +frame_id: '' and grasped_object_mask header seq: 130061 +stamp: + secs: 1734490552 + nsecs: 27054071 +frame_id: "camera_color_optical_frame" +[INFO] [1734490552.455184]: received mask_obs with header seq: 2119 +stamp: + secs: 1734490552 + nsecs: 458743572 +frame_id: '' and grasped_object_mask header seq: 130067 +stamp: + secs: 1734490552 + nsecs: 226891279 +frame_id: "camera_color_optical_frame" +[INFO] [1734490552.657081]: received mask_obs with header seq: 2120 +stamp: + secs: 1734490552 + nsecs: 660816431 +frame_id: '' and grasped_object_mask header seq: 130073 +stamp: + secs: 1734490552 + nsecs: 426727533 +frame_id: "camera_color_optical_frame" +[INFO] [1734490552.857090]: received mask_obs with header seq: 2121 +stamp: + secs: 1734490552 + nsecs: 860948324 +frame_id: '' and grasped_object_mask header seq: 130079 +stamp: + secs: 1734490552 + nsecs: 626563787 +frame_id: "camera_color_optical_frame" +[INFO] [1734490553.055507]: received mask_obs with header seq: 2122 +stamp: + secs: 1734490553 + nsecs: 59247732 +frame_id: '' and grasped_object_mask header seq: 130085 +stamp: + secs: 1734490552 + nsecs: 826400518 +frame_id: "camera_color_optical_frame" +[INFO] [1734490553.256879]: received mask_obs with header seq: 2123 +stamp: + secs: 1734490553 + nsecs: 260744333 +frame_id: '' and grasped_object_mask header seq: 130091 +stamp: + secs: 1734490553 + nsecs: 26242495 +frame_id: "camera_color_optical_frame" +[INFO] [1734490553.457077]: received mask_obs with header seq: 2124 +stamp: + secs: 1734490553 + nsecs: 460819244 +frame_id: '' and grasped_object_mask header seq: 130097 +stamp: + secs: 1734490553 + nsecs: 226084471 +frame_id: "camera_color_optical_frame" +[INFO] [1734490553.656219]: received mask_obs with header seq: 2125 +stamp: + secs: 1734490553 + nsecs: 659826517 +frame_id: '' and grasped_object_mask header seq: 130103 +stamp: + secs: 1734490553 + nsecs: 425927877 +frame_id: "camera_color_optical_frame" +[INFO] [1734490553.854256]: received mask_obs with header seq: 2126 +stamp: + secs: 1734490553 + nsecs: 857952356 +frame_id: '' and grasped_object_mask header seq: 130109 +stamp: + secs: 1734490553 + nsecs: 625770807 +frame_id: "camera_color_optical_frame" +[INFO] [1734490554.057269]: received mask_obs with header seq: 2127 +stamp: + secs: 1734490554 + nsecs: 60918569 +frame_id: '' and grasped_object_mask header seq: 130115 +stamp: + secs: 1734490553 + nsecs: 825613737 +frame_id: "camera_color_optical_frame" +[INFO] [1734490554.255906]: received mask_obs with header seq: 2128 +stamp: + secs: 1734490554 + nsecs: 259711503 +frame_id: '' and grasped_object_mask header seq: 130121 +stamp: + secs: 1734490554 + nsecs: 25457382 +frame_id: "camera_color_optical_frame" +[INFO] [1734490554.454594]: received mask_obs with header seq: 2129 +stamp: + secs: 1734490554 + nsecs: 458460092 +frame_id: '' and grasped_object_mask header seq: 130127 +stamp: + secs: 1734490554 + nsecs: 225301027 +frame_id: "camera_color_optical_frame" +[INFO] [1734490554.656617]: received mask_obs with header seq: 2130 +stamp: + secs: 1734490554 + nsecs: 659420967 +frame_id: '' and grasped_object_mask header seq: 130133 +stamp: + secs: 1734490554 + nsecs: 425144911 +frame_id: "camera_color_optical_frame" +[INFO] [1734490554.856617]: received mask_obs with header seq: 2131 +stamp: + secs: 1734490554 + nsecs: 860342025 +frame_id: '' and grasped_object_mask header seq: 130139 +stamp: + secs: 1734490554 + nsecs: 624990225 +frame_id: "camera_color_optical_frame" +[INFO] [1734490555.055284]: received mask_obs with header seq: 2132 +stamp: + secs: 1734490555 + nsecs: 59002876 +frame_id: '' and grasped_object_mask header seq: 130145 +stamp: + secs: 1734490554 + nsecs: 824833155 +frame_id: "camera_color_optical_frame" +[WARN] [1734490556.841713]: Plan exhausted +[INFO] [1734490555.256449]: received mask_obs with header seq: 2133 +stamp: + secs: 1734490555 + nsecs: 260080337 +frame_id: '' and grasped_object_mask header seq: 130151 +stamp: + secs: 1734490555 + nsecs: 24669886 +frame_id: "camera_color_optical_frame" +[INFO] [1734490555.455123]: received mask_obs with header seq: 2134 +stamp: + secs: 1734490555 + nsecs: 458895444 +frame_id: '' and grasped_object_mask header seq: 130157 +stamp: + secs: 1734490555 + nsecs: 224506378 +frame_id: "camera_color_optical_frame" +preempting episode +[INFO] [1734490555.551305]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1734490555.654847]: received mask_obs with header seq: 2135 +stamp: + secs: 1734490555 + nsecs: 658508300 +frame_id: '' and grasped_object_mask header seq: 130163 +stamp: + secs: 1734490555 + nsecs: 424342632 +frame_id: "camera_color_optical_frame" +[INFO] [1734490556.369471]: Stored episode 4. +[INFO] [1734490556.372656]: Saving video... +[INFO] [1734490556.792182]: Video saved! +Starting episode 4 +[INFO] [1734490556.853321]: resetting environment +[INFO] [1734490556.871173]: cleared current plan +[INFO] [1734490556.880654]: moving to home +[INFO] [1734490560.294308]: reached home +[INFO] [1734490560.302821]: reset action history +[INFO] [1734490563.109295]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1734490578.453786]: resetting environment +[INFO] [1734490578.476282]: cleared current plan +[INFO] [1734490578.482746]: moving to home +[INFO] [1734490579.488386]: reached home +[INFO] [1734490579.493275]: reset action history +[INFO] [1734490582.274404]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/episode_rosbags/episode_4_2024-12-17-21-56-22.bag +... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-77808.log +started roslaunch server http://158.130.50.37:33647/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [78714] +process[print_text-14]: started with pid [78715] +process[rosbag_record-15]: started with pid [78716] +started bagging! +[INFO] [1734490583.114307]: received mask_obs with header seq: 2136 +stamp: + secs: 1734490583 + nsecs: 117811679 +frame_id: '' and grasped_object_mask header seq: 130988 +stamp: + secs: 1734490582 + nsecs: 908037424 +frame_id: "camera_color_optical_frame" +[INFO] [1734490583.309057]: received mask_obs with header seq: 2137 +stamp: + secs: 1734490583 + nsecs: 312637329 +frame_id: '' and grasped_object_mask header seq: 130994 +stamp: + secs: 1734490583 + nsecs: 107652903 +frame_id: "camera_color_optical_frame" +[INFO] [1734490583.509572]: received mask_obs with header seq: 2138 +stamp: + secs: 1734490583 + nsecs: 513030767 +frame_id: '' and grasped_object_mask header seq: 131000 +stamp: + secs: 1734490583 + nsecs: 307257175 +frame_id: "camera_color_optical_frame" +[INFO] [1734490583.713777]: received mask_obs with header seq: 2139 +stamp: + secs: 1734490583 + nsecs: 717352151 +frame_id: '' and grasped_object_mask header seq: 131006 +stamp: + secs: 1734490583 + nsecs: 506849766 +frame_id: "camera_color_optical_frame" +[INFO] [1734490583.917855]: received mask_obs with header seq: 2140 +stamp: + secs: 1734490583 + nsecs: 921683311 +frame_id: '' and grasped_object_mask header seq: 131012 +stamp: + secs: 1734490583 + nsecs: 706432343 +frame_id: "camera_color_optical_frame" +[INFO] [1734490584.123987]: received mask_obs with header seq: 2141 +stamp: + secs: 1734490584 + nsecs: 127638101 +frame_id: '' and grasped_object_mask header seq: 131018 +stamp: + secs: 1734490583 + nsecs: 906019688 +frame_id: "camera_color_optical_frame" +[INFO] [1734490584.329152]: received mask_obs with header seq: 2142 +stamp: + secs: 1734490584 + nsecs: 332940816 +frame_id: '' and grasped_object_mask header seq: 131024 +stamp: + secs: 1734490584 + nsecs: 105641365 +frame_id: "camera_color_optical_frame" +[INFO] [1734490584.533576]: received mask_obs with header seq: 2143 +stamp: + secs: 1734490584 + nsecs: 536446094 +frame_id: '' and grasped_object_mask header seq: 131030 +stamp: + secs: 1734490584 + nsecs: 305251598 +frame_id: "camera_color_optical_frame" +[INFO] [1734490584.737453]: received mask_obs with header seq: 2144 +stamp: + secs: 1734490584 + nsecs: 741110563 +frame_id: '' and grasped_object_mask header seq: 131036 +stamp: + secs: 1734490584 + nsecs: 504851103 +frame_id: "camera_color_optical_frame" +[INFO] [1734490584.935995]: received mask_obs with header seq: 2145 +stamp: + secs: 1734490584 + nsecs: 939564228 +frame_id: '' and grasped_object_mask header seq: 131042 +stamp: + secs: 1734490584 + nsecs: 704438925 +frame_id: "camera_color_optical_frame" +[INFO] [1734490585.137046]: received mask_obs with header seq: 2146 +stamp: + secs: 1734490585 + nsecs: 140800952 +frame_id: '' and grasped_object_mask header seq: 131048 +stamp: + secs: 1734490584 + nsecs: 904031277 +frame_id: "camera_color_optical_frame" +[INFO] [1734490585.333861]: received mask_obs with header seq: 2147 +stamp: + secs: 1734490585 + nsecs: 337641477 +frame_id: '' and grasped_object_mask header seq: 131054 +stamp: + secs: 1734490585 + nsecs: 103649378 +frame_id: "camera_color_optical_frame" +[INFO] [1734490585.534265]: received mask_obs with header seq: 2148 +stamp: + secs: 1734490585 + nsecs: 537238121 +frame_id: '' and grasped_object_mask header seq: 131060 +stamp: + secs: 1734490585 + nsecs: 303256512 +frame_id: "camera_color_optical_frame" +[INFO] [1734490585.732607]: received mask_obs with header seq: 2149 +stamp: + secs: 1734490585 + nsecs: 736168861 +frame_id: '' and grasped_object_mask header seq: 131066 +stamp: + secs: 1734490585 + nsecs: 502851963 +frame_id: "camera_color_optical_frame" +[INFO] [1734490585.936679]: received mask_obs with header seq: 2150 +stamp: + secs: 1734490585 + nsecs: 940269708 +frame_id: '' and grasped_object_mask header seq: 131072 +stamp: + secs: 1734490585 + nsecs: 702436209 +frame_id: "camera_color_optical_frame" +[INFO] [1734490586.133570]: received mask_obs with header seq: 2151 +stamp: + secs: 1734490586 + nsecs: 137134790 +frame_id: '' and grasped_object_mask header seq: 131078 +stamp: + secs: 1734490585 + nsecs: 902024031 +frame_id: "camera_color_optical_frame" +[INFO] [1734490586.332202]: received mask_obs with header seq: 2152 +stamp: + secs: 1734490586 + nsecs: 335948705 +frame_id: '' and grasped_object_mask header seq: 131084 +stamp: + secs: 1734490586 + nsecs: 101642132 +frame_id: "camera_color_optical_frame" +[INFO] [1734490586.532192]: received mask_obs with header seq: 2153 +stamp: + secs: 1734490586 + nsecs: 535948753 +frame_id: '' and grasped_object_mask header seq: 131090 +stamp: + secs: 1734490586 + nsecs: 301247835 +frame_id: "camera_color_optical_frame" +[INFO] [1734490586.734647]: received mask_obs with header seq: 2154 +stamp: + secs: 1734490586 + nsecs: 738460063 +frame_id: '' and grasped_object_mask header seq: 131096 +stamp: + secs: 1734490586 + nsecs: 500841856 +frame_id: "camera_color_optical_frame" +[INFO] [1734490586.930200]: received mask_obs with header seq: 2155 +stamp: + secs: 1734490586 + nsecs: 933973312 +frame_id: '' and grasped_object_mask header seq: 131102 +stamp: + secs: 1734490586 + nsecs: 700424671 +frame_id: "camera_color_optical_frame" +[INFO] [1734490587.131986]: received mask_obs with header seq: 2156 +stamp: + secs: 1734490587 + nsecs: 135693073 +frame_id: '' and grasped_object_mask header seq: 131108 +stamp: + secs: 1734490586 + nsecs: 900011301 +frame_id: "camera_color_optical_frame" +[INFO] [1734490587.333589]: received mask_obs with header seq: 2157 +stamp: + secs: 1734490587 + nsecs: 337314367 +frame_id: '' and grasped_object_mask header seq: 131114 +stamp: + secs: 1734490587 + nsecs: 99629641 +frame_id: "camera_color_optical_frame" +[INFO] [1734490587.532206]: received mask_obs with header seq: 2158 +stamp: + secs: 1734490587 + nsecs: 536006689 +frame_id: '' and grasped_object_mask header seq: 131120 +stamp: + secs: 1734490587 + nsecs: 299237728 +frame_id: "camera_color_optical_frame" +[INFO] [1734490587.732607]: received mask_obs with header seq: 2159 +stamp: + secs: 1734490587 + nsecs: 736068248 +frame_id: '' and grasped_object_mask header seq: 131126 +stamp: + secs: 1734490587 + nsecs: 498833179 +frame_id: "camera_color_optical_frame" +[INFO] [1734490587.931263]: received mask_obs with header seq: 2160 +stamp: + secs: 1734490587 + nsecs: 934935569 +frame_id: '' and grasped_object_mask header seq: 131132 +stamp: + secs: 1734490587 + nsecs: 698417187 +frame_id: "camera_color_optical_frame" +[INFO] [1734490588.130608]: received mask_obs with header seq: 2161 +stamp: + secs: 1734490588 + nsecs: 134336948 +frame_id: '' and grasped_object_mask header seq: 131138 +stamp: + secs: 1734490587 + nsecs: 898003817 +frame_id: "camera_color_optical_frame" +[INFO] [1734490588.337499]: received mask_obs with header seq: 2162 +stamp: + secs: 1734490588 + nsecs: 341223955 +frame_id: '' and grasped_object_mask header seq: 131144 +stamp: + secs: 1734490588 + nsecs: 97617626 +frame_id: "camera_color_optical_frame" +[INFO] [1734490588.544910]: received mask_obs with header seq: 2163 +stamp: + secs: 1734490588 + nsecs: 548620462 +frame_id: '' and grasped_object_mask header seq: 131150 +stamp: + secs: 1734490588 + nsecs: 297219753 +frame_id: "camera_color_optical_frame" +[INFO] [1734490588.737477]: received mask_obs with header seq: 2164 +stamp: + secs: 1734490588 + nsecs: 741187334 +frame_id: '' and grasped_object_mask header seq: 131156 +stamp: + secs: 1734490588 + nsecs: 496810198 +frame_id: "camera_color_optical_frame" +[INFO] [1734490588.934209]: received mask_obs with header seq: 2165 +stamp: + secs: 1734490588 + nsecs: 937886238 +frame_id: '' and grasped_object_mask header seq: 131162 +stamp: + secs: 1734490588 + nsecs: 696390390 +frame_id: "camera_color_optical_frame" +[INFO] [1734490589.132544]: received mask_obs with header seq: 2166 +stamp: + secs: 1734490589 + nsecs: 135853290 +frame_id: '' and grasped_object_mask header seq: 131168 +stamp: + secs: 1734490588 + nsecs: 895973206 +frame_id: "camera_color_optical_frame" +[INFO] [1734490589.332468]: received mask_obs with header seq: 2167 +stamp: + secs: 1734490589 + nsecs: 335838317 +frame_id: '' and grasped_object_mask header seq: 131174 +stamp: + secs: 1734490589 + nsecs: 95587730 +frame_id: "camera_color_optical_frame" +[INFO] [1734490589.536875]: received mask_obs with header seq: 2168 +stamp: + secs: 1734490589 + nsecs: 540460348 +frame_id: '' and grasped_object_mask header seq: 131180 +stamp: + secs: 1734490589 + nsecs: 295190573 +frame_id: "camera_color_optical_frame" +[INFO] [1734490589.742541]: received mask_obs with header seq: 2169 +stamp: + secs: 1734490589 + nsecs: 746243953 +frame_id: '' and grasped_object_mask header seq: 131186 +stamp: + secs: 1734490589 + nsecs: 494781971 +frame_id: "camera_color_optical_frame" +[INFO] [1734490589.938070]: received mask_obs with header seq: 2170 +stamp: + secs: 1734490589 + nsecs: 941799640 +frame_id: '' and grasped_object_mask header seq: 131192 +stamp: + secs: 1734490589 + nsecs: 694361687 +frame_id: "camera_color_optical_frame" +[INFO] [1734490590.133970]: received mask_obs with header seq: 2171 +stamp: + secs: 1734490590 + nsecs: 137722253 +frame_id: '' and grasped_object_mask header seq: 131198 +stamp: + secs: 1734490589 + nsecs: 893945456 +frame_id: "camera_color_optical_frame" +[INFO] [1734490590.333535]: received mask_obs with header seq: 2172 +stamp: + secs: 1734490590 + nsecs: 337215185 +frame_id: '' and grasped_object_mask header seq: 131204 +stamp: + secs: 1734490590 + nsecs: 93560219 +frame_id: "camera_color_optical_frame" +[INFO] [1734490590.531558]: received mask_obs with header seq: 2173 +stamp: + secs: 1734490590 + nsecs: 535300016 +frame_id: '' and grasped_object_mask header seq: 131210 +stamp: + secs: 1734490590 + nsecs: 293163061 +frame_id: "camera_color_optical_frame" +[INFO] [1734490590.732053]: received mask_obs with header seq: 2174 +stamp: + secs: 1734490590 + nsecs: 735605716 +frame_id: '' and grasped_object_mask header seq: 131216 +stamp: + secs: 1734490590 + nsecs: 492754459 +frame_id: "camera_color_optical_frame" +[INFO] [1734490590.932062]: received mask_obs with header seq: 2175 +stamp: + secs: 1734490590 + nsecs: 935529708 +frame_id: '' and grasped_object_mask header seq: 131222 +stamp: + secs: 1734490590 + nsecs: 692333937 +frame_id: "camera_color_optical_frame" +[INFO] [1734490591.131439]: received mask_obs with header seq: 2176 +stamp: + secs: 1734490591 + nsecs: 134928226 +frame_id: '' and grasped_object_mask header seq: 131228 +stamp: + secs: 1734490590 + nsecs: 891929388 +frame_id: "camera_color_optical_frame" +[INFO] [1734490591.332804]: received mask_obs with header seq: 2177 +stamp: + secs: 1734490591 + nsecs: 335503339 +frame_id: '' and grasped_object_mask header seq: 131234 +stamp: + secs: 1734490591 + nsecs: 93683481 +frame_id: "camera_color_optical_frame" +[INFO] [1734490591.533934]: received mask_obs with header seq: 2178 +stamp: + secs: 1734490591 + nsecs: 537366390 +frame_id: '' and grasped_object_mask header seq: 131240 +stamp: + secs: 1734490591 + nsecs: 295684099 +frame_id: "camera_color_optical_frame" +[INFO] [1734490591.732689]: received mask_obs with header seq: 2179 +stamp: + secs: 1734490591 + nsecs: 735617876 +frame_id: '' and grasped_object_mask header seq: 131246 +stamp: + secs: 1734490591 + nsecs: 497760296 +frame_id: "camera_color_optical_frame" +[INFO] [1734490591.935121]: received mask_obs with header seq: 2180 +stamp: + secs: 1734490591 + nsecs: 938884496 +frame_id: '' and grasped_object_mask header seq: 131252 +stamp: + secs: 1734490591 + nsecs: 699912071 +frame_id: "camera_color_optical_frame" +[INFO] [1734490592.143012]: received mask_obs with header seq: 2181 +stamp: + secs: 1734490592 + nsecs: 146224498 +frame_id: '' and grasped_object_mask header seq: 131258 +stamp: + secs: 1734490591 + nsecs: 902140141 +frame_id: "camera_color_optical_frame" +[INFO] [1734490592.337080]: received mask_obs with header seq: 2182 +stamp: + secs: 1734490592 + nsecs: 340583086 +frame_id: '' and grasped_object_mask header seq: 131264 +stamp: + secs: 1734490592 + nsecs: 101769447 +frame_id: "camera_color_optical_frame" +[INFO] [1734490592.533177]: received mask_obs with header seq: 2183 +stamp: + secs: 1734490592 + nsecs: 536620616 +frame_id: '' and grasped_object_mask header seq: 131270 +stamp: + secs: 1734490592 + nsecs: 301147699 +frame_id: "camera_color_optical_frame" +[INFO] [1734490592.737231]: received mask_obs with header seq: 2184 +stamp: + secs: 1734490592 + nsecs: 740522146 +frame_id: '' and grasped_object_mask header seq: 131276 +stamp: + secs: 1734490592 + nsecs: 500501633 +frame_id: "camera_color_optical_frame" +[INFO] [1734490592.936678]: received mask_obs with header seq: 2185 +stamp: + secs: 1734490592 + nsecs: 940224885 +frame_id: '' and grasped_object_mask header seq: 131282 +stamp: + secs: 1734490592 + nsecs: 699832678 +frame_id: "camera_color_optical_frame" +[INFO] [1734490593.132146]: received mask_obs with header seq: 2186 +stamp: + secs: 1734490593 + nsecs: 135794162 +frame_id: '' and grasped_object_mask header seq: 131288 +stamp: + secs: 1734490592 + nsecs: 899140835 +frame_id: "camera_color_optical_frame" +[INFO] [1734490593.324001]: received mask_obs with header seq: 2187 +stamp: + secs: 1734490593 + nsecs: 327686548 +frame_id: '' and grasped_object_mask header seq: 131290 +stamp: + secs: 1734490592 + nsecs: 965912819 +frame_id: "camera_color_optical_frame" +[INFO] [1734490593.524942]: received mask_obs with header seq: 2188 +stamp: + secs: 1734490593 + nsecs: 528632164 +frame_id: '' and grasped_object_mask header seq: 131300 +stamp: + secs: 1734490593 + nsecs: 300457954 +frame_id: "camera_color_optical_frame" +[INFO] [1734490593.724570]: received mask_obs with header seq: 2189 +stamp: + secs: 1734490593 + nsecs: 728029012 +frame_id: '' and grasped_object_mask header seq: 131306 +stamp: + secs: 1734490593 + nsecs: 501286983 +frame_id: "camera_color_optical_frame" +[INFO] [1734490593.931969]: received mask_obs with header seq: 2190 +stamp: + secs: 1734490593 + nsecs: 935661554 +frame_id: '' and grasped_object_mask header seq: 131312 +stamp: + secs: 1734490593 + nsecs: 702190876 +frame_id: "camera_color_optical_frame" +[INFO] [1734490594.150554]: received mask_obs with header seq: 2191 +stamp: + secs: 1734490594 + nsecs: 153904199 +frame_id: '' and grasped_object_mask header seq: 131318 +stamp: + secs: 1734490593 + nsecs: 903170347 +frame_id: "camera_color_optical_frame" +[INFO] [1734490594.342325]: received mask_obs with header seq: 2192 +stamp: + secs: 1734490594 + nsecs: 343700170 +frame_id: '' and grasped_object_mask header seq: 131324 +stamp: + secs: 1734490594 + nsecs: 102988482 +frame_id: "camera_color_optical_frame" +[INFO] [1734490594.532675]: received mask_obs with header seq: 2193 +stamp: + secs: 1734490594 + nsecs: 536213874 +frame_id: '' and grasped_object_mask header seq: 131330 +stamp: + secs: 1734490594 + nsecs: 302587748 +frame_id: "camera_color_optical_frame" +[INFO] [1734490594.731117]: received mask_obs with header seq: 2194 +stamp: + secs: 1734490594 + nsecs: 734782457 +frame_id: '' and grasped_object_mask header seq: 131336 +stamp: + secs: 1734490594 + nsecs: 502174854 +frame_id: "camera_color_optical_frame" +[INFO] [1734490594.924946]: received mask_obs with header seq: 2195 +stamp: + secs: 1734490594 + nsecs: 928667068 +frame_id: '' and grasped_object_mask header seq: 131342 +stamp: + secs: 1734490594 + nsecs: 701751232 +frame_id: "camera_color_optical_frame" +[INFO] [1734490595.125342]: received mask_obs with header seq: 2196 +stamp: + secs: 1734490595 + nsecs: 128903627 +frame_id: '' and grasped_object_mask header seq: 131348 +stamp: + secs: 1734490594 + nsecs: 901314497 +frame_id: "camera_color_optical_frame" +[INFO] [1734490595.325122]: received mask_obs with header seq: 2197 +stamp: + secs: 1734490595 + nsecs: 328766822 +frame_id: '' and grasped_object_mask header seq: 131354 +stamp: + secs: 1734490595 + nsecs: 100909948 +frame_id: "camera_color_optical_frame" +[INFO] [1734490595.524944]: received mask_obs with header seq: 2198 +stamp: + secs: 1734490595 + nsecs: 528267860 +frame_id: '' and grasped_object_mask header seq: 131360 +stamp: + secs: 1734490595 + nsecs: 300508261 +frame_id: "camera_color_optical_frame" +[INFO] [1734490595.723127]: received mask_obs with header seq: 2199 +stamp: + secs: 1734490595 + nsecs: 726871490 +frame_id: '' and grasped_object_mask header seq: 131366 +stamp: + secs: 1734490595 + nsecs: 500094891 +frame_id: "camera_color_optical_frame" +[INFO] [1734490595.924582]: received mask_obs with header seq: 2200 +stamp: + secs: 1734490595 + nsecs: 928237199 +frame_id: '' and grasped_object_mask header seq: 131372 +stamp: + secs: 1734490595 + nsecs: 699669600 +frame_id: "camera_color_optical_frame" +[INFO] [1734490596.124247]: received mask_obs with header seq: 2201 +stamp: + secs: 1734490596 + nsecs: 127907752 +frame_id: '' and grasped_object_mask header seq: 131378 +stamp: + secs: 1734490595 + nsecs: 899232149 +frame_id: "camera_color_optical_frame" +[INFO] [1734490596.326460]: received mask_obs with header seq: 2202 +stamp: + secs: 1734490596 + nsecs: 329791784 +frame_id: '' and grasped_object_mask header seq: 131384 +stamp: + secs: 1734490596 + nsecs: 98823786 +frame_id: "camera_color_optical_frame" +[INFO] [1734490596.525831]: received mask_obs with header seq: 2203 +stamp: + secs: 1734490596 + nsecs: 529433965 +frame_id: '' and grasped_object_mask header seq: 131390 +stamp: + secs: 1734490596 + nsecs: 298415661 +frame_id: "camera_color_optical_frame" +[INFO] [1734490596.725163]: received mask_obs with header seq: 2204 +stamp: + secs: 1734490596 + nsecs: 728793859 +frame_id: '' and grasped_object_mask header seq: 131396 +stamp: + secs: 1734490596 + nsecs: 497995615 +frame_id: "camera_color_optical_frame" +[INFO] [1734490596.925943]: received mask_obs with header seq: 2205 +stamp: + secs: 1734490596 + nsecs: 929662704 +frame_id: '' and grasped_object_mask header seq: 131402 +stamp: + secs: 1734490596 + nsecs: 697563171 +frame_id: "camera_color_optical_frame" +[INFO] [1734490597.126072]: received mask_obs with header seq: 2206 +stamp: + secs: 1734490597 + nsecs: 129636287 +frame_id: '' and grasped_object_mask header seq: 131408 +stamp: + secs: 1734490596 + nsecs: 897118568 +frame_id: "camera_color_optical_frame" +[INFO] [1734490597.328339]: received mask_obs with header seq: 2207 +stamp: + secs: 1734490597 + nsecs: 331699371 +frame_id: '' and grasped_object_mask header seq: 131414 +stamp: + secs: 1734490597 + nsecs: 96708536 +frame_id: "camera_color_optical_frame" +[INFO] [1734490597.523967]: received mask_obs with header seq: 2208 +stamp: + secs: 1734490597 + nsecs: 527680158 +frame_id: '' and grasped_object_mask header seq: 131420 +stamp: + secs: 1734490597 + nsecs: 296303749 +frame_id: "camera_color_optical_frame" +[INFO] [1734490597.726094]: received mask_obs with header seq: 2209 +stamp: + secs: 1734490597 + nsecs: 729793071 +frame_id: '' and grasped_object_mask header seq: 131426 +stamp: + secs: 1734490597 + nsecs: 495885611 +frame_id: "camera_color_optical_frame" +[INFO] [1734490597.933251]: received mask_obs with header seq: 2210 +stamp: + secs: 1734490597 + nsecs: 936908006 +frame_id: '' and grasped_object_mask header seq: 131432 +stamp: + secs: 1734490597 + nsecs: 695455074 +frame_id: "camera_color_optical_frame" +[INFO] [1734490598.129173]: received mask_obs with header seq: 2211 +stamp: + secs: 1734490598 + nsecs: 132756948 +frame_id: '' and grasped_object_mask header seq: 131438 +stamp: + secs: 1734490597 + nsecs: 895012379 +frame_id: "camera_color_optical_frame" +[INFO] [1734490598.328216]: received mask_obs with header seq: 2212 +stamp: + secs: 1734490598 + nsecs: 331652641 +frame_id: '' and grasped_object_mask header seq: 131444 +stamp: + secs: 1734490598 + nsecs: 94607592 +frame_id: "camera_color_optical_frame" +[INFO] [1734490598.523099]: received mask_obs with header seq: 2213 +stamp: + secs: 1734490598 + nsecs: 526822090 +frame_id: '' and grasped_object_mask header seq: 131450 +stamp: + secs: 1734490598 + nsecs: 294209003 +frame_id: "camera_color_optical_frame" +[INFO] [1734490598.726771]: received mask_obs with header seq: 2214 +stamp: + secs: 1734490598 + nsecs: 730146646 +frame_id: '' and grasped_object_mask header seq: 131456 +stamp: + secs: 1734490598 + nsecs: 493799448 +frame_id: "camera_color_optical_frame" +[INFO] [1734490598.927766]: received mask_obs with header seq: 2215 +stamp: + secs: 1734490598 + nsecs: 931231975 +frame_id: '' and grasped_object_mask header seq: 131462 +stamp: + secs: 1734490598 + nsecs: 693377018 +frame_id: "camera_color_optical_frame" +[INFO] [1734490599.126199]: received mask_obs with header seq: 2216 +stamp: + secs: 1734490599 + nsecs: 129703283 +frame_id: '' and grasped_object_mask header seq: 131468 +stamp: + secs: 1734490598 + nsecs: 892942667 +frame_id: "camera_color_optical_frame" +[INFO] [1734490599.326481]: received mask_obs with header seq: 2217 +stamp: + secs: 1734490599 + nsecs: 330220460 +frame_id: '' and grasped_object_mask header seq: 131474 +stamp: + secs: 1734490599 + nsecs: 92538118 +frame_id: "camera_color_optical_frame" +[INFO] [1734490599.527888]: received mask_obs with header seq: 2218 +stamp: + secs: 1734490599 + nsecs: 531403541 +frame_id: '' and grasped_object_mask header seq: 131480 +stamp: + secs: 1734490599 + nsecs: 292136669 +frame_id: "camera_color_optical_frame" +[INFO] [1734490599.737769]: received mask_obs with header seq: 2219 +stamp: + secs: 1734490599 + nsecs: 740563154 +frame_id: '' and grasped_object_mask header seq: 131486 +stamp: + secs: 1734490599 + nsecs: 491723299 +frame_id: "camera_color_optical_frame" +[INFO] [1734490599.932141]: received mask_obs with header seq: 2220 +stamp: + secs: 1734490599 + nsecs: 935684442 +frame_id: '' and grasped_object_mask header seq: 131492 +stamp: + secs: 1734490599 + nsecs: 691298723 +frame_id: "camera_color_optical_frame" +[INFO] [1734490600.125544]: received mask_obs with header seq: 2221 +stamp: + secs: 1734490600 + nsecs: 129273176 +frame_id: '' and grasped_object_mask header seq: 131498 +stamp: + secs: 1734490599 + nsecs: 890861511 +frame_id: "camera_color_optical_frame" +[INFO] [1734490600.338987]: received mask_obs with header seq: 2222 +stamp: + secs: 1734490600 + nsecs: 341288805 +frame_id: '' and grasped_object_mask header seq: 131504 +stamp: + secs: 1734490600 + nsecs: 90462446 +frame_id: "camera_color_optical_frame" +[INFO] [1734490600.530891]: received mask_obs with header seq: 2223 +stamp: + secs: 1734490600 + nsecs: 534600019 +frame_id: '' and grasped_object_mask header seq: 131510 +stamp: + secs: 1734490600 + nsecs: 290070295 +frame_id: "camera_color_optical_frame" +[INFO] [1734490600.727342]: received mask_obs with header seq: 2224 +stamp: + secs: 1734490600 + nsecs: 730972766 +frame_id: '' and grasped_object_mask header seq: 131516 +stamp: + secs: 1734490600 + nsecs: 489666462 +frame_id: "camera_color_optical_frame" +[INFO] [1734490600.924220]: received mask_obs with header seq: 2225 +stamp: + secs: 1734490600 + nsecs: 927942037 +frame_id: '' and grasped_object_mask header seq: 131522 +stamp: + secs: 1734490600 + nsecs: 689251423 +frame_id: "camera_color_optical_frame" +[INFO] [1734490601.126378]: received mask_obs with header seq: 2226 +stamp: + secs: 1734490601 + nsecs: 130102872 +frame_id: '' and grasped_object_mask header seq: 131528 +stamp: + secs: 1734490600 + nsecs: 888825655 +frame_id: "camera_color_optical_frame" +[INFO] [1734490601.340142]: received mask_obs with header seq: 2227 +stamp: + secs: 1734490601 + nsecs: 343854904 +frame_id: '' and grasped_object_mask header seq: 131534 +stamp: + secs: 1734490601 + nsecs: 88425398 +frame_id: "camera_color_optical_frame" +[INFO] [1734490601.531533]: received mask_obs with header seq: 2228 +stamp: + secs: 1734490601 + nsecs: 535257101 +frame_id: '' and grasped_object_mask header seq: 131540 +stamp: + secs: 1734490601 + nsecs: 288027048 +frame_id: "camera_color_optical_frame" +[INFO] [1734490601.725347]: received mask_obs with header seq: 2229 +stamp: + secs: 1734490601 + nsecs: 729060888 +frame_id: '' and grasped_object_mask header seq: 131546 +stamp: + secs: 1734490601 + nsecs: 487617254 +frame_id: "camera_color_optical_frame" +[INFO] [1734490601.925113]: received mask_obs with header seq: 2230 +stamp: + secs: 1734490601 + nsecs: 928538322 +frame_id: '' and grasped_object_mask header seq: 131552 +stamp: + secs: 1734490601 + nsecs: 687195539 +frame_id: "camera_color_optical_frame" +[INFO] [1734490602.125183]: received mask_obs with header seq: 2231 +stamp: + secs: 1734490602 + nsecs: 128931999 +frame_id: '' and grasped_object_mask header seq: 131558 +stamp: + secs: 1734490601 + nsecs: 886762142 +frame_id: "camera_color_optical_frame" +[INFO] [1734490602.324899]: received mask_obs with header seq: 2232 +stamp: + secs: 1734490602 + nsecs: 328634023 +frame_id: '' and grasped_object_mask header seq: 131564 +stamp: + secs: 1734490602 + nsecs: 86364985 +frame_id: "camera_color_optical_frame" +[INFO] [1734490602.525387]: received mask_obs with header seq: 2233 +stamp: + secs: 1734490602 + nsecs: 529101610 +frame_id: '' and grasped_object_mask header seq: 131570 +stamp: + secs: 1734490602 + nsecs: 285973072 +frame_id: "camera_color_optical_frame" +[INFO] [1734490602.725999]: received mask_obs with header seq: 2234 +stamp: + secs: 1734490602 + nsecs: 729610443 +frame_id: '' and grasped_object_mask header seq: 131576 +stamp: + secs: 1734490602 + nsecs: 485569477 +frame_id: "camera_color_optical_frame" +[INFO] [1734490602.924119]: received mask_obs with header seq: 2235 +stamp: + secs: 1734490602 + nsecs: 927850008 +frame_id: '' and grasped_object_mask header seq: 131582 +stamp: + secs: 1734490602 + nsecs: 685154676 +frame_id: "camera_color_optical_frame" +[INFO] [1734490603.125494]: received mask_obs with header seq: 2236 +stamp: + secs: 1734490603 + nsecs: 129176616 +frame_id: '' and grasped_object_mask header seq: 131588 +stamp: + secs: 1734490602 + nsecs: 884727955 +frame_id: "camera_color_optical_frame" +[INFO] [1734490603.324318]: received mask_obs with header seq: 2237 +stamp: + secs: 1734490603 + nsecs: 327882528 +frame_id: '' and grasped_object_mask header seq: 131594 +stamp: + secs: 1734490603 + nsecs: 84329128 +frame_id: "camera_color_optical_frame" +[INFO] [1734490603.523137]: received mask_obs with header seq: 2238 +stamp: + secs: 1734490603 + nsecs: 526890516 +frame_id: '' and grasped_object_mask header seq: 131600 +stamp: + secs: 1734490603 + nsecs: 283933640 +frame_id: "camera_color_optical_frame" +[INFO] [1734490603.724285]: received mask_obs with header seq: 2239 +stamp: + secs: 1734490603 + nsecs: 727970600 +frame_id: '' and grasped_object_mask header seq: 131606 +stamp: + secs: 1734490603 + nsecs: 483527184 +frame_id: "camera_color_optical_frame" +[INFO] [1734490603.923387]: received mask_obs with header seq: 2240 +stamp: + secs: 1734490603 + nsecs: 927028894 +frame_id: '' and grasped_object_mask header seq: 131612 +stamp: + secs: 1734490603 + nsecs: 683108091 +frame_id: "camera_color_optical_frame" +[INFO] [1734490604.124236]: received mask_obs with header seq: 2241 +stamp: + secs: 1734490604 + nsecs: 127868175 +frame_id: '' and grasped_object_mask header seq: 131618 +stamp: + secs: 1734490603 + nsecs: 882677555 +frame_id: "camera_color_optical_frame" +[INFO] [1734490604.326760]: received mask_obs with header seq: 2242 +stamp: + secs: 1734490604 + nsecs: 330458879 +frame_id: '' and grasped_object_mask header seq: 131624 +stamp: + secs: 1734490604 + nsecs: 82273245 +frame_id: "camera_color_optical_frame" +[INFO] [1734490604.524348]: received mask_obs with header seq: 2243 +stamp: + secs: 1734490604 + nsecs: 527984380 +frame_id: '' and grasped_object_mask header seq: 131630 +stamp: + secs: 1734490604 + nsecs: 281871557 +frame_id: "camera_color_optical_frame" +[INFO] [1734490604.724463]: received mask_obs with header seq: 2244 +stamp: + secs: 1734490604 + nsecs: 728150367 +frame_id: '' and grasped_object_mask header seq: 131636 +stamp: + secs: 1734490604 + nsecs: 481457949 +frame_id: "camera_color_optical_frame" +[INFO] [1734490604.923816]: received mask_obs with header seq: 2245 +stamp: + secs: 1734490604 + nsecs: 927419185 +frame_id: '' and grasped_object_mask header seq: 131642 +stamp: + secs: 1734490604 + nsecs: 681033373 +frame_id: "camera_color_optical_frame" +[INFO] [1734490605.124758]: received mask_obs with header seq: 2246 +stamp: + secs: 1734490605 + nsecs: 128454446 +frame_id: '' and grasped_object_mask header seq: 131648 +stamp: + secs: 1734490604 + nsecs: 880595684 +frame_id: "camera_color_optical_frame" +[INFO] [1734490605.343400]: received mask_obs with header seq: 2247 +stamp: + secs: 1734490605 + nsecs: 346822261 +frame_id: '' and grasped_object_mask header seq: 131654 +stamp: + secs: 1734490605 + nsecs: 80193281 +frame_id: "camera_color_optical_frame" +[INFO] [1734490605.544378]: received mask_obs with header seq: 2248 +stamp: + secs: 1734490605 + nsecs: 548095464 +frame_id: '' and grasped_object_mask header seq: 131661 +stamp: + secs: 1734490605 + nsecs: 313064814 +frame_id: "camera_color_optical_frame" +[INFO] [1734490605.741678]: received mask_obs with header seq: 2249 +stamp: + secs: 1734490605 + nsecs: 745431661 +frame_id: '' and grasped_object_mask header seq: 131667 +stamp: + secs: 1734490605 + nsecs: 512655497 +frame_id: "camera_color_optical_frame" +[INFO] [1734490605.943542]: received mask_obs with header seq: 2250 +stamp: + secs: 1734490605 + nsecs: 947253227 +frame_id: '' and grasped_object_mask header seq: 131673 +stamp: + secs: 1734490605 + nsecs: 712234259 +frame_id: "camera_color_optical_frame" +[INFO] [1734490606.138534]: received mask_obs with header seq: 2251 +stamp: + secs: 1734490606 + nsecs: 142141103 +frame_id: '' and grasped_object_mask header seq: 131679 +stamp: + secs: 1734490605 + nsecs: 911801577 +frame_id: "camera_color_optical_frame" +[INFO] [1734490606.335211]: received mask_obs with header seq: 2252 +stamp: + secs: 1734490606 + nsecs: 338758230 +frame_id: '' and grasped_object_mask header seq: 131684 +stamp: + secs: 1734490606 + nsecs: 78138590 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 104377963 for nearest: 1734490606382117488 - target: 1734490606277739525 at idx 249 +[INFO] [1734490606.526053]: received mask_obs with header seq: 2253 +stamp: + secs: 1734490606 + nsecs: 529502153 +frame_id: '' and grasped_object_mask header seq: 131690 +stamp: + secs: 1734490606 + nsecs: 277739525 +frame_id: "camera_color_optical_frame" +[INFO] [1734490606.724909]: received mask_obs with header seq: 2254 +stamp: + secs: 1734490606 + nsecs: 728369235 +frame_id: '' and grasped_object_mask header seq: 131696 +stamp: + secs: 1734490606 + nsecs: 477328539 +frame_id: "camera_color_optical_frame" +[INFO] [1734490606.924149]: received mask_obs with header seq: 2255 +stamp: + secs: 1734490606 + nsecs: 927907943 +frame_id: '' and grasped_object_mask header seq: 131702 +stamp: + secs: 1734490606 + nsecs: 676905632 +frame_id: "camera_color_optical_frame" +[INFO] [1734490607.124980]: received mask_obs with header seq: 2256 +stamp: + secs: 1734490607 + nsecs: 128546714 +frame_id: '' and grasped_object_mask header seq: 131708 +stamp: + secs: 1734490606 + nsecs: 876470804 +frame_id: "camera_color_optical_frame" +[INFO] [1734490607.328451]: received mask_obs with header seq: 2257 +stamp: + secs: 1734490607 + nsecs: 331885814 +frame_id: '' and grasped_object_mask header seq: 131714 +stamp: + secs: 1734490607 + nsecs: 76984406 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 104084833 for nearest: 1734490607382041557 - target: 1734490607277956724 at idx 249 +[INFO] [1734490607.526995]: received mask_obs with header seq: 2258 +stamp: + secs: 1734490607 + nsecs: 530688762 +frame_id: '' and grasped_object_mask header seq: 131720 +stamp: + secs: 1734490607 + nsecs: 277956724 +frame_id: "camera_color_optical_frame" +[INFO] [1734490607.726763]: received mask_obs with header seq: 2259 +stamp: + secs: 1734490607 + nsecs: 730438947 +frame_id: '' and grasped_object_mask header seq: 131726 +stamp: + secs: 1734490607 + nsecs: 479017496 +frame_id: "camera_color_optical_frame" +[INFO] [1734490607.926157]: received mask_obs with header seq: 2260 +stamp: + secs: 1734490607 + nsecs: 929798364 +frame_id: '' and grasped_object_mask header seq: 131732 +stamp: + secs: 1734490607 + nsecs: 680166721 +frame_id: "camera_color_optical_frame" +[INFO] [1734490608.127413]: received mask_obs with header seq: 2261 +stamp: + secs: 1734490608 + nsecs: 130857706 +frame_id: '' and grasped_object_mask header seq: 131738 +stamp: + secs: 1734490607 + nsecs: 881404161 +frame_id: "camera_color_optical_frame" +[INFO] [1734490608.327346]: received mask_obs with header seq: 2262 +stamp: + secs: 1734490608 + nsecs: 330922842 +frame_id: '' and grasped_object_mask header seq: 131744 +stamp: + secs: 1734490608 + nsecs: 81700087 +frame_id: "camera_color_optical_frame" +[INFO] [1734490608.531559]: received mask_obs with header seq: 2263 +stamp: + secs: 1734490608 + nsecs: 534620046 +frame_id: '' and grasped_object_mask header seq: 131750 +stamp: + secs: 1734490608 + nsecs: 281546354 +frame_id: "camera_color_optical_frame" +[INFO] [1734490608.729017]: received mask_obs with header seq: 2264 +stamp: + secs: 1734490608 + nsecs: 732240438 +frame_id: '' and grasped_object_mask header seq: 131756 +stamp: + secs: 1734490608 + nsecs: 481392145 +frame_id: "camera_color_optical_frame" +[INFO] [1734490608.923670]: received mask_obs with header seq: 2265 +stamp: + secs: 1734490608 + nsecs: 927375555 +frame_id: '' and grasped_object_mask header seq: 131762 +stamp: + secs: 1734490608 + nsecs: 681237936 +frame_id: "camera_color_optical_frame" +[INFO] [1734490609.125054]: received mask_obs with header seq: 2266 +stamp: + secs: 1734490609 + nsecs: 128777742 +frame_id: '' and grasped_object_mask header seq: 131768 +stamp: + secs: 1734490608 + nsecs: 881084204 +frame_id: "camera_color_optical_frame" +[INFO] [1734490609.343505]: received mask_obs with header seq: 2267 +stamp: + secs: 1734490609 + nsecs: 346809625 +frame_id: '' and grasped_object_mask header seq: 131775 +stamp: + secs: 1734490609 + nsecs: 114230156 +frame_id: "camera_color_optical_frame" +[INFO] [1734490609.388476]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1734490609.545072]: received mask_obs with header seq: 2268 +stamp: + secs: 1734490609 + nsecs: 548797845 +frame_id: '' and grasped_object_mask header seq: 131781 +stamp: + secs: 1734490609 + nsecs: 314066172 +frame_id: "camera_color_optical_frame" +[INFO] [1734490610.154630]: Stored episode 5. +[INFO] [1734490610.161205]: Saving video... +[INFO] [1734490610.397165]: Video saved! +[INFO] [1734490610.438393]: resetting environment +[INFO] [1734490610.477713]: cleared current plan +[INFO] [1734490610.484712]: moving to home +[INFO] [1734490614.091765]: reached home +[INFO] [1734490614.095583]: reset action history +[INFO] [1734490616.931647]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt b/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json b/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e2dbeccb9f62593d4b9dc14e6e9ac31dc037214a --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-18T02:52:46.558919", + "startedAt": "2024-12-18T02:52:46.119968", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3572.842125, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4517.241, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4536.345, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 392.8499336242676 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json b/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f32bdf9d20873d76e5e1fd94751bfb90f8716662 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "dbd1eabcbea39a4495283b2f6184ef4a084c2c15f963d7e5faa8bbd22cbd06b6", "size": 1175932, "path": "media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4"}, "global_step": 624, "_timestamp": 1734490624.39174, "_runtime": 258.2612040042877, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "42cf14600b8d2071849cb7efe24d11a32495dab3954997e2c2a53233f5841b6b", "size": 1179891, "path": "media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/2_eval": {"_type": "video-file", "sha256": "5329c53c34465295af0a20968ecc659fb9b1f59c569d2d70bc8ca0b2b92cb0b7", "size": 1184804, "path": "media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "f7f27b273940874a00d0beb9ee861cc2707f80e7a7361acb0344fc73e8ad9ba8", "size": 929140, "path": "media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "08fea80871bf35a0fb30f5931132d38f7ca937e097e8e5057847c2ba0b1ff3ee", "size": 1194519, "path": "media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4"}, "_wandb": {"runtime": 257}} \ No newline at end of file diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug-internal.log b/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d8c3cc254357ef2c26fd0bc3d5b4da4209dde30f --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug-internal.log @@ -0,0 +1,753 @@ +2024-12-17 21:52:46,130 INFO StreamThr :78001 [internal.py:wandb_internal():85] W&B internal server running at pid: 78001, started at: 2024-12-17 21:52:46.129837 +2024-12-17 21:52:46,131 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status +2024-12-17 21:52:46,144 INFO WriterThread:78001 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb +2024-12-17 21:52:46,144 DEBUG SenderThread:78001 [sender.py:send():379] send: header +2024-12-17 21:52:46,152 DEBUG SenderThread:78001 [sender.py:send():379] send: run +2024-12-17 21:52:46,475 INFO SenderThread:78001 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files +2024-12-17 21:52:46,475 INFO SenderThread:78001 [sender.py:_start_run_threads():1188] run started: g6nsr00w with start time 1734490366.130536 +2024-12-17 21:52:46,494 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: check_version +2024-12-17 21:52:46,494 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: check_version +2024-12-17 21:52:46,540 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: run_start +2024-12-17 21:52:46,544 DEBUG HandlerThread:78001 [system_info.py:__init__():26] System info init +2024-12-17 21:52:46,544 DEBUG HandlerThread:78001 [system_info.py:__init__():41] System info init done +2024-12-17 21:52:46,544 INFO HandlerThread:78001 [system_monitor.py:start():194] Starting system monitor +2024-12-17 21:52:46,545 INFO SystemMonitor:78001 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-17 21:52:46,545 INFO HandlerThread:78001 [system_monitor.py:probe():214] Collecting system info +2024-12-17 21:52:46,545 INFO SystemMonitor:78001 [interfaces.py:start():188] Started cpu monitoring +2024-12-17 21:52:46,546 INFO SystemMonitor:78001 [interfaces.py:start():188] Started disk monitoring +2024-12-17 21:52:46,546 INFO SystemMonitor:78001 [interfaces.py:start():188] Started gpu monitoring +2024-12-17 21:52:46,547 INFO SystemMonitor:78001 [interfaces.py:start():188] Started memory monitoring +2024-12-17 21:52:46,547 INFO SystemMonitor:78001 [interfaces.py:start():188] Started network monitoring +2024-12-17 21:52:46,558 DEBUG HandlerThread:78001 [system_info.py:probe():152] Probing system +2024-12-17 21:52:46,560 DEBUG HandlerThread:78001 [system_info.py:_probe_git():137] Probing git +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_info.py:_probe_git():145] Probing git done +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_info.py:probe():200] Probing system done +2024-12-17 21:52:46,564 DEBUG HandlerThread:78001 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:52:46.558919', 'startedAt': '2024-12-18T02:52:46.119968', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3572.842125, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4517.241, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4536.345, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 392.8499336242676}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-17 21:52:46,565 INFO HandlerThread:78001 [system_monitor.py:probe():224] Finished collecting system info +2024-12-17 21:52:46,565 INFO HandlerThread:78001 [system_monitor.py:probe():227] Publishing system info +2024-12-17 21:52:46,565 DEBUG HandlerThread:78001 [system_info.py:_save_code():44] Saving code +2024-12-17 21:52:46,567 DEBUG HandlerThread:78001 [system_info.py:_save_code():67] Saving code done +2024-12-17 21:52:46,567 DEBUG HandlerThread:78001 [system_info.py:_save_patches():84] Saving git patches +2024-12-17 21:52:46,676 DEBUG HandlerThread:78001 [system_info.py:_save_patches():126] Saving git patches done +2024-12-17 21:52:46,677 INFO HandlerThread:78001 [system_monitor.py:probe():229] Finished publishing system info +2024-12-17 21:52:46,680 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-17 21:52:46,680 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-17 21:52:46,907 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: python_packages +2024-12-17 21:52:46,907 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: python_packages +2024-12-17 21:52:46,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:52:46,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:52:46,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:46,910 INFO HandlerThread:78001 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/tb" + save: true +} +control { + relay_id: "394677d9b6aa4385a9b0ebca7a0e61ef" +} +_info { + stream_id: "g6nsr00w" +} + +2024-12-17 21:52:46,911 DEBUG HandlerThread:78001 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-17 21:52:46,935 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/cmvh8ay6-wandb-metadata.json +2024-12-17 21:52:46,944 INFO wandb-upload_1:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/iupp8rxj-code/FISH/eval_robot.py +2024-12-17 21:52:46,953 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/fmlykv68-diff.patch +2024-12-17 21:52:47,056 DEBUG SenderThread:78001 [sender.py:send():379] send: telemetry +2024-12-17 21:52:47,056 DEBUG SenderThread:78001 [sender.py:send():379] send: tbrecord +2024-12-17 21:52:47,057 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:52:47,057 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 with policy live +2024-12-17 21:52:47,234 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/xsh0mhuy-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:52:47,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code +2024-12-17 21:52:47,478 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH +2024-12-17 21:52:47,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:48,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:49,476 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:49,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:50,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:51,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:52:51,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:52,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:53,720 DEBUG SenderThread:78001 [sender.py:send():379] send: config +2024-12-17 21:52:53,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:54,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:55,477 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:55,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:56,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:52:56,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:57,478 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:57,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:58,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:52:59,479 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:52:59,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:00,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:01,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:01,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:01,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:02,060 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:02,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:03,480 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:03,910 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:04,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:05,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:06,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:07,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:07,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:08,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:09,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:10,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:11,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:12,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:12,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:13,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:14,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:15,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:16,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:16,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:16,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:17,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:18,033 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:18,486 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml +2024-12-17 21:53:18,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:19,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:20,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:21,487 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:21,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:22,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:23,488 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:23,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:23,913 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:24,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:25,488 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:25,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:26,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:27,489 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:27,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:28,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:29,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:29,490 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:29,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:30,914 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:31,490 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:31,908 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:31,908 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:31,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:32,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:33,491 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:33,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:34,165 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:34,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:35,491 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:35,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:36,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:37,492 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:37,915 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:38,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:39,330 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:39,492 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:39,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:40,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:41,493 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:41,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:42,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:43,494 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:43,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:44,529 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:44,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:45,494 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:45,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:46,548 DEBUG SystemMonitor:78001 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-17 21:53:46,548 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:53:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:53:46,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:53:46,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:47,495 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:47,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:48,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:49,495 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:49,548 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:49,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:50,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:51,496 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:51,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:52,319 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:53:52,319 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 with policy now +2024-12-17 21:53:52,319 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:53:52,320 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:53:52,320 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:53:52,320 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:53:52,496 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos +2024-12-17 21:53:52,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media +2024-12-17 21:53:52,542 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rdsdzo9c-media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,775 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/ydvne3ig-media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:53:52,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:53,497 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:53,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:54,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:55,331 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:53:55,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:56,918 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:57,498 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:57,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:58,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:53:59,499 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:53:59,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:00,689 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:00,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:01,500 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:01,741 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/kizcyfj9-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:01,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:01,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:01,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:02,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:03,500 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:03,919 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:04,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:05,834 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:05,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:06,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:07,502 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:07,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:08,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:09,502 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:09,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:10,869 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:10,920 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:11,503 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:11,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:12,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:13,504 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:13,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:14,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:15,505 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:15,871 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:15,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:16,549 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:54:16,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:16,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:16,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:17,505 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:17,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:18,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:19,506 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:19,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:20,871 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:20,922 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:21,507 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:21,922 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:22,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:23,507 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:23,923 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:24,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:25,508 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:25,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:26,078 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:26,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:27,509 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:27,924 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:28,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:29,510 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:29,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:30,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:31,274 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:31,510 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:31,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:31,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:31,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:32,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:33,409 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:54:33,409 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 with policy now +2024-12-17 21:54:33,412 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:54:33,412 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:54:33,413 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:54:33,413 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:54:33,511 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,512 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:54:33,641 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/hvxa2wmt-media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,894 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/hllcqyuf-media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:54:33,930 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:34,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:35,512 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:35,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:36,427 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:36,933 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:37,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:38,933 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:39,513 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:39,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:40,931 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:41,514 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:41,861 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:41,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:42,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:43,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:44,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:45,932 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:46,549 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:54:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:54:46,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:54:46,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:47,040 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:47,312 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:54:47,312 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:54:47,312 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:54:47,516 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:54:47,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:48,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:49,517 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:49,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:50,934 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:51,517 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:51,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:52,197 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:52,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:53,518 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:53,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:54,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:55,519 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:55,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:56,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:57,220 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:54:57,520 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:57,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:58,935 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:54:59,520 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:54:59,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:00,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:01,521 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:01,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:01,909 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:01,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:02,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:02,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:03,522 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:03,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:04,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:05,523 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:05,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:06,936 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:07,423 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:07,523 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:07,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:08,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:09,524 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:09,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:10,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:11,525 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:11,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:12,610 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:12,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:13,525 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:13,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:14,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:15,526 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:15,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:16,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:55:16,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:16,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:16,937 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:17,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:17,610 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:17,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:18,608 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:55:18,608 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 with policy now +2024-12-17 21:55:18,609 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:55:18,609 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:18,609 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:18,610 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:18,851 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rquo0y7d-media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:18,941 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:19,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:19,529 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:19,530 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:19,530 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:55:19,800 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/11petlc4-media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:55:19,941 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:20,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:21,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:22,944 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:23,285 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:23,531 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:23,942 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:24,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:25,531 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:25,943 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:26,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:27,532 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:27,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:28,917 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:28,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:29,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:30,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:31,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:31,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:31,946 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:32,154 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:32,154 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:32,155 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:32,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:32,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:55:32,777 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/2vpbc8if-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:55:32,954 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:33,534 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:33,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:34,645 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:34,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:35,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:36,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:37,535 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:37,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:38,947 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:39,536 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:39,665 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:39,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:40,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:41,537 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:41,948 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:42,949 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:43,537 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:43,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:44,867 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:44,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:45,538 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:45,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:46,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:55:46,909 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:55:46,910 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:55:46,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:47,539 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:47,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:48,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:49,540 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:49,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:50,059 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:50,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:51,541 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:51,950 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:52,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:53,541 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:53,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:54,951 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:55,256 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:55:55,542 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:55,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:56,842 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:55:56,842 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 with policy now +2024-12-17 21:55:56,847 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:55:56,847 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:55:56,847 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:55:56,848 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:55:56,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:57,057 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/i1alqjlt-media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,543 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:55:57,793 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/zvcyis0o-media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:55:57,955 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:58,966 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:55:59,967 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:00,295 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:00,968 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:01,544 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:01,912 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:01,912 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:01,971 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:02,965 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:03,545 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:03,969 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:04,971 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:05,977 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:06,157 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:06,972 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:07,974 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:08,982 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:09,976 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:10,980 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:11,925 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:11,983 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:12,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:13,986 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:14,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:15,984 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:16,550 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:56:16,916 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:16,916 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:16,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:17,034 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:17,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:18,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:56:18,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:19,211 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:56:19,211 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:56:19,212 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:56:19,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:19,550 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:56:19,985 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:20,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:21,551 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:21,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:22,275 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:22,986 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:23,552 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:23,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:24,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:25,552 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:25,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:26,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:27,334 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:27,553 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:27,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:28,987 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:29,554 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:29,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:30,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:31,555 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:31,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:31,911 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:31,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:32,337 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:32,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:33,556 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:33,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:34,988 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:35,556 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:35,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:36,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:37,525 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:37,557 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:37,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:38,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:39,561 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:39,989 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:40,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:41,566 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:41,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:42,525 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:42,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:43,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:44,568 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:44,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:45,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:46,551 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:56:46,569 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:46,911 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:56:46,911 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:56:46,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:47,527 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:47,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:48,570 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:48,990 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:49,995 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:50,434 DEBUG SenderThread:78001 [sender.py:send():379] send: files +2024-12-17 21:56:50,435 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 with policy now +2024-12-17 21:56:50,437 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 21:56:50,437 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:56:50,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:56:50,438 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:56:50,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:50,572 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval +2024-12-17 21:56:50,640 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/rw2u2d5b-media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:50,894 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/z1u4c5tn-media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:56:51,006 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:52,023 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:52,571 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:52,929 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:52,999 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:54,021 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:55,000 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:56,013 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:56,573 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:57,001 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:57,973 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:56:58,012 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:56:58,573 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:56:59,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:00,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:01,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:01,921 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 21:57:01,921 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: stop_status +2024-12-17 21:57:02,015 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:03,016 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:03,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:04,016 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:04,437 DEBUG SenderThread:78001 [sender.py:send():379] send: exit +2024-12-17 21:57:04,437 INFO SenderThread:78001 [sender.py:send_exit():586] handling exit code: 0 +2024-12-17 21:57:04,437 INFO SenderThread:78001 [sender.py:send_exit():588] handling runtime: 257 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_exit():594] send defer +2024-12-17 21:57:04,438 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,438 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-17 21:57:04,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 1 +2024-12-17 21:57:04,438 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,438 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-17 21:57:04,438 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-17 21:57:04,438 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 2 +2024-12-17 21:57:04,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [system_monitor.py:finish():203] Stopping system monitor +2024-12-17 21:57:04,439 DEBUG SystemMonitor:78001 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined cpu monitor +2024-12-17 21:57:04,439 DEBUG SystemMonitor:78001 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-17 21:57:04,439 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined disk monitor +2024-12-17 21:57:04,486 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined gpu monitor +2024-12-17 21:57:04,486 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined memory monitor +2024-12-17 21:57:04,487 INFO HandlerThread:78001 [interfaces.py:finish():200] Joined network monitor +2024-12-17 21:57:04,487 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,487 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-17 21:57:04,487 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 3 +2024-12-17 21:57:04,487 DEBUG SenderThread:78001 [sender.py:send():379] send: stats +2024-12-17 21:57:04,487 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,487 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-17 21:57:04,488 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:04,488 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-17 21:57:04,488 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 4 +2024-12-17 21:57:04,488 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:04,488 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-17 21:57:04,575 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:04,576 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:04,810 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /tmp/tmpbfuocajxwandb/ypj7c7k4-events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:06,576 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:10,491 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,491 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: keepalive +2024-12-17 21:57:10,491 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-17 21:57:10,492 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 5 +2024-12-17 21:57:10,492 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:57:10,492 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:57:10,492 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,493 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-17 21:57:10,493 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,493 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:10,493 DEBUG SenderThread:78001 [sender.py:send():379] send: history +2024-12-17 21:57:10,494 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: summary_record +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,497 DEBUG SenderThread:78001 [sender.py:send():379] send: summary +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 21:57:10,497 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,497 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 6 +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,498 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-17 21:57:10,498 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 7 +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 21:57:10,498 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:10,498 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-17 21:57:10,498 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:10,498 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-17 21:57:10,577 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:11,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:13,429 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 8 +2024-12-17 21:57:13,429 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:13,429 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:13,429 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-17 21:57:13,430 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:13,430 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-17 21:57:13,430 INFO SenderThread:78001 [job_builder.py:build():440] Attempting to build job artifact +2024-12-17 21:57:13,432 INFO SenderThread:78001 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-17 21:57:13,439 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:13,451 INFO SenderThread:78001 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-17 21:57:13,461 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 9 +2024-12-17 21:57:13,462 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:13,462 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:13,462 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-17 21:57:13,462 DEBUG SenderThread:78001 [sender.py:send():379] send: artifact +2024-12-17 21:57:13,579 INFO Thread-12 :78001 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:14,015 INFO wandb-upload_0:78001 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpkt9zbjlr +2024-12-17 21:57:14,015 INFO wandb-upload_1:78001 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpxnpvbcq3 +2024-12-17 21:57:14,229 INFO wandb-upload_2:78001 [upload_job.py:push():88] Uploaded file /tmp/tmpz2cro5k3/wandb-job.json +2024-12-17 21:57:14,440 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:14,830 INFO SenderThread:78001 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzMzNjA0Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzIyNjY3MQ==', 'versionIndex': 379}}} +2024-12-17 21:57:14,830 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:14,830 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-17 21:57:14,830 INFO SenderThread:78001 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-17 21:57:15,580 INFO SenderThread:78001 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml config.yaml +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 events.out.tfevents.1734490366.leonmkim-ROG-Strix-G15CS-G15CS.77808.0 +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json wandb-summary.json +2024-12-17 21:57:15,581 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt requirements.txt +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-metadata.json wandb-metadata.json +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log output.log +2024-12-17 21:57:15,583 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/diff.patch diff.patch +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 media/videos/eval/2_eval_2_5329c53c34465295af0a.mp4 +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 media/videos/eval/4_eval_4_08fea80871bf35a0fb30.mp4 +2024-12-17 21:57:15,589 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 media/videos/eval/1_eval_1_42cf14600b8d2071849c.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 media/videos/eval/0_eval_0_dbd1eabcbea39a449528.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 media/videos/eval/3_eval_3_f7f27b273940874a00d0.mp4 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 10 +2024-12-17 21:57:15,590 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:15,590 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:15,590 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-17 21:57:15,590 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:15,590 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-17 21:57:15,590 INFO SenderThread:78001 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:57:15,773 INFO wandb-upload_0:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/config.yaml +2024-12-17 21:57:15,794 INFO wandb-upload_3:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/requirements.txt +2024-12-17 21:57:15,812 INFO wandb-upload_2:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/output.log +2024-12-17 21:57:15,813 INFO wandb-upload_1:78001 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/files/wandb-summary.json +2024-12-17 21:57:16,013 INFO Thread-11 (_thread_body):78001 [sender.py:transition_state():614] send defer: 11 +2024-12-17 21:57:16,014 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,014 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-17 21:57:16,014 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,014 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-17 21:57:16,014 INFO SenderThread:78001 [file_pusher.py:join():175] waiting for file pusher +2024-12-17 21:57:16,017 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 12 +2024-12-17 21:57:16,017 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,017 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-17 21:57:16,017 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,017 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-17 21:57:16,018 INFO SenderThread:78001 [file_stream.py:finish():601] file stream finish called +2024-12-17 21:57:16,223 INFO SenderThread:78001 [file_stream.py:finish():605] file stream finish is done +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 13 +2024-12-17 21:57:16,223 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,223 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-17 21:57:16,223 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-17 21:57:16,223 INFO SenderThread:78001 [sender.py:transition_state():614] send defer: 14 +2024-12-17 21:57:16,223 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: defer +2024-12-17 21:57:16,223 INFO HandlerThread:78001 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send():379] send: final +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send():379] send: footer +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: defer +2024-12-17 21:57:16,224 INFO SenderThread:78001 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-17 21:57:16,224 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:16,224 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:16,224 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: server_info +2024-12-17 21:57:16,225 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 21:57:16,225 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: server_info +2024-12-17 21:57:16,226 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: get_summary +2024-12-17 21:57:16,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-17 21:57:16,227 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 21:57:16,292 DEBUG SenderThread:78001 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-17 21:57:16,293 INFO MainThread:78001 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-17 21:57:16,293 DEBUG HandlerThread:78001 [handler.py:handle_request():158] handle_request: shutdown +2024-12-17 21:57:16,293 INFO HandlerThread:78001 [handler.py:finish():882] shutting down handler +2024-12-17 21:57:17,225 INFO WriterThread:78001 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb +2024-12-17 21:57:17,293 INFO SenderThread:78001 [sender.py:finish():1615] shutting down sender +2024-12-17 21:57:17,293 INFO SenderThread:78001 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 21:57:17,293 INFO SenderThread:78001 [file_pusher.py:join():175] waiting for file pusher diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug.log b/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0bc576de422da9b1767af1fc9f6047cc614a3e8c --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Configure stats pid to 77808 +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-17 21:52:46,121 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/settings +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug.log +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/wandb/run-20241217_215246-g6nsr00w/logs/debug-internal.log +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():569] calling init triggers +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '523_0', 'wandb_notes': '523_0_req_603_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240'} +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():619] starting backend +2024-12-17 21:52:46,122 INFO MainThread:77808 [wandb_init.py:init():623] setting up manager +2024-12-17 21:52:46,129 INFO MainThread:77808 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-17 21:52:46,130 INFO MainThread:77808 [wandb_init.py:init():631] backend started and connected +2024-12-17 21:52:46,143 INFO MainThread:77808 [wandb_init.py:init():720] updated telemetry +2024-12-17 21:52:46,149 INFO MainThread:77808 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-17 21:52:46,493 INFO MainThread:77808 [wandb_run.py:_on_init():2435] communicating current version +2024-12-17 21:52:46,533 INFO MainThread:77808 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-17 21:52:46,534 INFO MainThread:77808 [wandb_init.py:init():804] starting run threads in backend +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_console_start():2413] atexit reg +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-17 21:52:46,908 INFO MainThread:77808 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-17 21:52:46,909 INFO MainThread:77808 [wandb_init.py:init():847] run started, returning control to user process +2024-12-17 21:52:46,910 INFO MainThread:77808 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/523_0/215240/tb, True +2024-12-17 21:52:53,719 INFO MainThread:77808 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2024-12-17 21:57:17,307 WARNING MsgRouterThr:77808 [router.py:message_loop():77] message_loop has been closed diff --git a/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb b/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb new file mode 100644 index 0000000000000000000000000000000000000000..53a82e45e855c098f92ea02bd18349ad7a5f4931 --- /dev/null +++ b/215240/wandb/run-20241217_215246-g6nsr00w/run-g6nsr00w.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a284d6fa8687db1f2df27da0558049a208d0a10a3648f1a807c898eff2e7496 +size 269608 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b623619d311f1bac4c2f706503a27335968fb838 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '523_0' +wandb_notes: 523_0_req_603_0 +eval: false +true_action_history: false +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- fowlers_twodim +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 0 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1