diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..2855ce2bcbadedbfe6ffa2f0b558817b7cb0014e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 filter=lfs diff=lfs merge=lfs -text +205123/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205451/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 filter=lfs diff=lfs merge=lfs -text +205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag filter=lfs diff=lfs merge=lfs -text +214032/episode_rosbags/episode_2_2024-12-10-21-42-51.rrd filter=lfs diff=lfs merge=lfs -text +214032/episode_rosbags/episode_0_2024-12-10-21-41-00.rrd filter=lfs diff=lfs merge=lfs -text +214032/episode_rosbags/episode_1_2024-12-10-21-41-44.rrd filter=lfs diff=lfs merge=lfs -text +214032/episode_rosbags/episode_4_2024-12-10-21-44-14.rrd filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 filter=lfs diff=lfs merge=lfs -text +214032/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 filter=lfs diff=lfs merge=lfs -text +214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 filter=lfs diff=lfs merge=lfs -text +214032/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +214032/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +214032/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +214032/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/204954/.hydra/config.yaml b/204954/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..800e8f2818ebbf3474b49aa980471298b50b023c --- /dev/null +++ b/204954/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: true +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/204954/.hydra/hydra.yaml b/204954/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e92487441d7301fe6ad1b6f4cba09308d7a195e4 --- /dev/null +++ b/204954/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/204954 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/204954/.hydra/overrides.yaml b/204954/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/204954/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/204954/eval_policy.log b/204954/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..d8e8142a772fba0fca5753479b26935514abd13e --- /dev/null +++ b/204954/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-10 20:49:54,427][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:49:54,430][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-10 20:49:55,731][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/205123/.hydra/config.yaml b/205123/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..800e8f2818ebbf3474b49aa980471298b50b023c --- /dev/null +++ b/205123/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: true +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205123/.hydra/hydra.yaml b/205123/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..230ea74ebddf0ac02bc447a4c2aef0ccea594039 --- /dev/null +++ b/205123/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205123/.hydra/overrides.yaml b/205123/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205123/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205123/episode_rosbags/aligned_depth_to_color_K.npy b/205123/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205123/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205123/episode_rosbags/cam_tf_world.npy b/205123/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/205123/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/205123/episode_rosbags/color_K.npy b/205123/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205123/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205123/episode_rosbags/depth_K.npy b/205123/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/205123/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/205123/eval_robot.log b/205123/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..5f8a2c3d56fe3ec84989bbe12b83ac1e62524ebb --- /dev/null +++ b/205123/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 20:51:23,588][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:51:23,591][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/205123/eval_video/0_eval.mp4 b/205123/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af52de142576ede297f9ed272b30ca90055285c3 --- /dev/null +++ b/205123/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10 +size 942048 diff --git a/205123/tb/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 b/205123/tb/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 new file mode 100644 index 0000000000000000000000000000000000000000..9b53794634b977ed21ecf0bed25b1492c0476d9a --- /dev/null +++ b/205123/tb/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19a2cb159a40df6149928fa963c721dd4fb49916ec660df1f61fffc949c49ec +size 88 diff --git a/205123/wandb/debug-internal.log b/205123/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..778b889c4c425bdaf3eb6b38f70268482f9cc6a8 --- /dev/null +++ b/205123/wandb/debug-internal.log @@ -0,0 +1,351 @@ +2024-12-10 20:51:30,311 INFO StreamThr :597014 [internal.py:wandb_internal():85] W&B internal server running at pid: 597014, started at: 2024-12-10 20:51:30.311157 +2024-12-10 20:51:30,312 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:51:30,325 INFO WriterThread:597014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb +2024-12-10 20:51:30,325 DEBUG SenderThread:597014 [sender.py:send():379] send: header +2024-12-10 20:51:30,336 DEBUG SenderThread:597014 [sender.py:send():379] send: run +2024-12-10 20:51:30,645 INFO SenderThread:597014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files +2024-12-10 20:51:30,645 INFO SenderThread:597014 [sender.py:_start_run_threads():1188] run started: svysfc56 with start time 1733881890.312299 +2024-12-10 20:51:30,656 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:51:30,656 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:51:30,743 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:51:30,748 DEBUG HandlerThread:597014 [system_info.py:__init__():26] System info init +2024-12-10 20:51:30,748 DEBUG HandlerThread:597014 [system_info.py:__init__():41] System info init done +2024-12-10 20:51:30,748 INFO HandlerThread:597014 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:51:30,748 INFO SystemMonitor:597014 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:51:30,749 INFO HandlerThread:597014 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:51:30,749 INFO SystemMonitor:597014 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:51:30,750 INFO SystemMonitor:597014 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:51:30,750 INFO SystemMonitor:597014 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:51:30,751 INFO SystemMonitor:597014 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:51:30,751 INFO SystemMonitor:597014 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:51:30,761 DEBUG HandlerThread:597014 [system_info.py:probe():152] Probing system +2024-12-10 20:51:30,762 DEBUG HandlerThread:597014 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_info.py:probe():200] Probing system done +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:51:30.761221', 'startedAt': '2024-12-11T01:51:30.302896', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.7015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 364.1780319213867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:51:30,768 INFO HandlerThread:597014 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:51:30,768 INFO HandlerThread:597014 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:51:30,768 DEBUG HandlerThread:597014 [system_info.py:_save_code():44] Saving code +2024-12-10 20:51:30,769 DEBUG HandlerThread:597014 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:51:30,769 DEBUG HandlerThread:597014 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:51:30,832 DEBUG HandlerThread:597014 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:51:30,832 INFO HandlerThread:597014 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:51:30,836 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:51:31,124 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/tiz5ae9w-diff.patch +2024-12-10 20:51:31,126 INFO wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/p6o167mp-wandb-metadata.json +2024-12-10 20:51:31,143 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:51:31,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:51:31,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:51:31,145 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:51:31,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:31,147 INFO HandlerThread:597014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb" + save: true +} +control { + relay_id: "42f7013eb6ef4a01b21dd778abc8c686" +} +_info { + stream_id: "svysfc56" +} + +2024-12-10 20:51:31,147 DEBUG HandlerThread:597014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:51:31,173 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/9ln43j5e-code/FISH/eval_robot.py +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: telemetry +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: tbrecord +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:51:31,218 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 with policy live +2024-12-10 20:51:31,415 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/08emxf8k-events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code +2024-12-10 20:51:32,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:33,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:33,646 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:34,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:35,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:36,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:36,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:37,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:38,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:39,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:40,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:41,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:41,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:42,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:43,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:44,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:45,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:46,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:51:46,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:51:46,186 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:46,300 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:46,908 DEBUG SenderThread:597014 [sender.py:send():379] send: config +2024-12-10 20:51:47,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:48,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:49,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:49,651 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:50,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:51,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:51,335 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:52,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:53,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:53,652 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:54,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:55,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:56,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:57,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:57,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:57,654 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:58,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:59,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:00,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:01,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:01,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:01,189 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:02,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:02,300 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:02,655 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml +2024-12-10 20:52:03,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:04,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:05,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:06,182 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:07,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:07,657 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:08,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:08,194 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:09,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:09,657 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:10,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:11,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:11,658 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:12,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:13,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:13,219 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:13,659 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:14,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:15,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:15,659 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:16,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:16,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:16,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:17,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:17,660 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:18,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:18,404 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:19,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:19,661 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:20,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:21,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:21,661 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:22,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:23,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:23,407 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:23,662 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:24,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:25,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:25,662 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:26,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:27,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:27,663 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:28,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:28,607 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:29,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:29,664 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:30,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:30,752 DEBUG SystemMonitor:597014 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:52:30,752 DEBUG SenderThread:597014 [sender.py:send():379] send: stats +2024-12-10 20:52:31,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:31,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:31,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:31,666 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:32,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:33,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:33,667 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:33,908 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:34,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:34,667 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:35,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:35,668 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:36,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:37,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:38,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:38,668 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:39,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:39,371 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:40,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:41,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:42,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:43,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:44,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:44,374 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:52:44,375 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 with policy now +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send():379] send: history +2024-12-10 20:52:44,375 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:44,377 DEBUG SenderThread:597014 [sender.py:send():379] send: exit +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():588] handling runtime: 73 +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():594] send defer +2024-12-10 20:52:44,379 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,379 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:52:44,380 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:52:44,380 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:52:44,380 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:52:44,380 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:52:44,381 DEBUG SystemMonitor:597014 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:52:44,381 DEBUG SystemMonitor:597014 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:52:44,381 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:52:44,382 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send():379] send: stats +2024-12-10 20:52:44,440 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,440 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:52:44,441 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,441 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:52:44,670 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media +2024-12-10 20:52:44,696 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/6w704qfx-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:45,030 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/kk0w0e5k-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:45,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,157 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:52:51,157 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:51,157 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,157 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send():379] send: summary +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:51,158 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,158 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:52:51,158 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,158 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:52:51,159 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,159 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:52:51,375 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:51,673 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:51,921 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:52:51,922 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:51,922 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,922 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:52:51,922 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,922 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:52:51,922 INFO SenderThread:597014 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:52:51,924 INFO SenderThread:597014 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:52:51,942 INFO SenderThread:597014 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:52:51,951 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:52:51,952 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,952 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:52:51,952 DEBUG SenderThread:597014 [sender.py:send():379] send: artifact +2024-12-10 20:52:52,376 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:52,626 INFO wandb-upload_0:597014 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmptoffctew +2024-12-10 20:52:52,673 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:52,858 INFO wandb-upload_2:597014 [upload_job.py:push():88] Uploaded file /tmp/tmpemx2d4yl/wandb-job.json +2024-12-10 20:52:52,880 INFO wandb-upload_1:597014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpc2y7tffk +2024-12-10 20:52:53,418 INFO SenderThread:597014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM1MTcyNjQ1OA==', 'versionIndex': 335}}} +2024-12-10 20:52:53,418 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:53,418 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:52:53,418 INFO SenderThread:597014 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml config.yaml +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json wandb-summary.json +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt requirements.txt +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log output.log +2024-12-10 20:52:53,676 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch diff.patch +2024-12-10 20:52:53,680 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:52:53,680 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:53,680 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:52:53,680 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:53,681 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:53,681 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:52:53,681 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:53,681 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:52:53,681 INFO SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:52:53,872 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt +2024-12-10 20:52:53,888 INFO wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml +2024-12-10 20:52:53,899 INFO wandb-upload_3:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:53,910 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:54,110 INFO Thread-11 (_thread_body):597014 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:52:54,110 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,111 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:52:54,111 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,111 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:52:54,111 INFO SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:52:54,111 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:52:54,111 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,111 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:52:54,112 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,112 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:52:54,112 INFO SenderThread:597014 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:52:54,317 INFO SenderThread:597014 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:52:54,317 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:52:54,318 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,318 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:52:54,318 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,318 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:52:54,318 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:52:54,318 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,318 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send():379] send: final +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send():379] send: footer +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,319 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:52:54,319 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:54,320 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:54,320 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:52:54,320 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:54,378 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:54,378 INFO MainThread:597014 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:52:54,378 INFO MainThread:597014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:52:54,379 INFO MainThread:597014 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:52:54,379 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:52:54,379 INFO HandlerThread:597014 [handler.py:finish():882] shutting down handler +2024-12-10 20:52:55,322 INFO WriterThread:597014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb +2024-12-10 20:52:55,378 INFO SenderThread:597014 [sender.py:finish():1615] shutting down sender +2024-12-10 20:52:55,378 INFO SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:52:55,378 INFO SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher diff --git a/205123/wandb/debug.log b/205123/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..10052a0fe4a55bedce5410671ea78339d38a1ac4 --- /dev/null +++ b/205123/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Configure stats pid to 596892 +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/settings +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': True}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():619] starting backend +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():623] setting up manager +2024-12-10 20:51:30,310 INFO MainThread:596892 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:51:30,312 INFO MainThread:596892 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:51:30,324 INFO MainThread:596892 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:51:30,334 INFO MainThread:596892 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:51:30,656 INFO MainThread:596892 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:51:30,737 INFO MainThread:596892 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:51:30,737 INFO MainThread:596892 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:51:31,144 INFO MainThread:596892 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:51:31,147 INFO MainThread:596892 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:51:31,147 INFO MainThread:596892 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb, True +2024-12-10 20:51:46,908 INFO MainThread:596892 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:52:55,379 WARNING MsgRouterThr:596892 [router.py:message_loop():77] message_loop has been closed diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py b/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml b/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f70d53740a2eb8979c3529c3e8cd046a27a5cfb9 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: true +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733881890 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch b/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e4d5c5ae6c67ab1e40fb5ca6e7d0f088ca1962fa --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch @@ -0,0 +1,25 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..c42ff82 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 b/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af52de142576ede297f9ed272b30ca90055285c3 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10 +size 942048 diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/output.log b/205123/wandb/run-20241210_205130-svysfc56/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..88450b583eb80105ca3a70107caa85bd93cdad38 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/output.log @@ -0,0 +1,1049 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733881906.908934]: resetting environment +[INFO] [1733881906.913976]: cleared current plan +[INFO] [1733881906.915491]: moving to home +[INFO] [1733881908.619161]: reached home +[INFO] [1733881908.620458]: reset action history +[INFO] [1733881911.334703]: environment reset +Starting episode 0 +[INFO] [1733881911.338240]: resetting environment +[INFO] [1733881911.346125]: cleared current plan +[INFO] [1733881911.347839]: moving to home +[INFO] [1733881912.349890]: reached home +[INFO] [1733881912.351231]: reset action history +[INFO] [1733881915.058954]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733881926.140345]: resetting environment +[INFO] [1733881926.145656]: cleared current plan +[INFO] [1733881926.150768]: moving to home +[INFO] [1733881927.153098]: reached home +[INFO] [1733881927.154883]: reset action history +[INFO] [1733881929.863341]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/episode_rosbags/episode_0_2024-12-10-20-52-09.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-596892.log +started roslaunch server http://158.130.50.37:44627/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [597118] +process[print_text-2]: started with pid [597119] +process[rosbag_record-3]: started with pid [597143] +started bagging! +[INFO] [1733881930.601407]: received mask_obs with header seq: 1 +stamp: + secs: 1733881930 + nsecs: 598197937 +frame_id: '' and grasped_object_mask header seq: 7743 +stamp: + secs: 1733881930 + nsecs: 373498678 +frame_id: "camera_color_optical_frame" +[INFO] [1733881930.790134]: received mask_obs with header seq: 2 +stamp: + secs: 1733881930 + nsecs: 787627696 +frame_id: '' and grasped_object_mask header seq: 7749 +stamp: + secs: 1733881930 + nsecs: 573334217 +frame_id: "camera_color_optical_frame" +[INFO] [1733881930.989544]: received mask_obs with header seq: 3 +stamp: + secs: 1733881930 + nsecs: 987669944 +frame_id: '' and grasped_object_mask header seq: 7755 +stamp: + secs: 1733881930 + nsecs: 773170948 +frame_id: "camera_color_optical_frame" +[INFO] [1733881931.186965]: received mask_obs with header seq: 4 +stamp: + secs: 1733881931 + nsecs: 185687780 +frame_id: '' and grasped_object_mask header seq: 7761 +stamp: + secs: 1733881930 + nsecs: 973006487 +frame_id: "camera_color_optical_frame" +[INFO] [1733881931.405436]: received mask_obs with header seq: 5 +stamp: + secs: 1733881931 + nsecs: 404456853 +frame_id: '' and grasped_object_mask header seq: 7767 +stamp: + secs: 1733881931 + nsecs: 172842264 +frame_id: "camera_color_optical_frame" +[INFO] [1733881931.606265]: received mask_obs with header seq: 6 +stamp: + secs: 1733881931 + nsecs: 605320453 +frame_id: '' and grasped_object_mask header seq: 7773 +stamp: + secs: 1733881931 + nsecs: 372677803 +frame_id: "camera_color_optical_frame" +[INFO] [1733881931.810085]: received mask_obs with header seq: 7 +stamp: + secs: 1733881931 + nsecs: 809148073 +frame_id: '' and grasped_object_mask header seq: 7779 +stamp: + secs: 1733881931 + nsecs: 572512865 +frame_id: "camera_color_optical_frame" +[INFO] [1733881932.000967]: received mask_obs with header seq: 8 +stamp: + secs: 1733881932 + nsecs: 77486 +frame_id: '' and grasped_object_mask header seq: 7785 +stamp: + secs: 1733881931 + nsecs: 772349596 +frame_id: "camera_color_optical_frame" +[INFO] [1733881932.200750]: received mask_obs with header seq: 9 +stamp: + secs: 1733881932 + nsecs: 199798583 +frame_id: '' and grasped_object_mask header seq: 7791 +stamp: + secs: 1733881931 + nsecs: 972187757 +frame_id: "camera_color_optical_frame" +[INFO] [1733881932.404178]: received mask_obs with header seq: 10 +stamp: + secs: 1733881932 + nsecs: 403140544 +frame_id: '' and grasped_object_mask header seq: 7797 +stamp: + secs: 1733881932 + nsecs: 172024965 +frame_id: "camera_color_optical_frame" +[INFO] [1733881932.603047]: received mask_obs with header seq: 11 +stamp: + secs: 1733881932 + nsecs: 602175712 +frame_id: '' and grasped_object_mask header seq: 7803 +stamp: + secs: 1733881932 + nsecs: 371861935 +frame_id: "camera_color_optical_frame" +[INFO] [1733881932.802278]: received mask_obs with header seq: 12 +stamp: + secs: 1733881932 + nsecs: 801258563 +frame_id: '' and grasped_object_mask header seq: 7809 +stamp: + secs: 1733881932 + nsecs: 571698666 +frame_id: "camera_color_optical_frame" +[INFO] [1733881933.018084]: received mask_obs with header seq: 13 +stamp: + secs: 1733881933 + nsecs: 17174005 +frame_id: '' and grasped_object_mask header seq: 7815 +stamp: + secs: 1733881932 + nsecs: 771533251 +frame_id: "camera_color_optical_frame" +[INFO] [1733881933.218519]: received mask_obs with header seq: 14 +stamp: + secs: 1733881933 + nsecs: 217555284 +frame_id: '' and grasped_object_mask header seq: 7821 +stamp: + secs: 1733881932 + nsecs: 971366405 +frame_id: "camera_color_optical_frame" +[INFO] [1733881933.411083]: received mask_obs with header seq: 15 +stamp: + secs: 1733881933 + nsecs: 410149812 +frame_id: '' and grasped_object_mask header seq: 7827 +stamp: + secs: 1733881933 + nsecs: 171200514 +frame_id: "camera_color_optical_frame" +[INFO] [1733881933.616644]: received mask_obs with header seq: 16 +stamp: + secs: 1733881933 + nsecs: 615602254 +frame_id: '' and grasped_object_mask header seq: 7833 +stamp: + secs: 1733881933 + nsecs: 371033192 +frame_id: "camera_color_optical_frame" +[INFO] [1733881933.810750]: received mask_obs with header seq: 17 +stamp: + secs: 1733881933 + nsecs: 809497833 +frame_id: '' and grasped_object_mask header seq: 7839 +stamp: + secs: 1733881933 + nsecs: 570865870 +frame_id: "camera_color_optical_frame" +[INFO] [1733881934.011081]: received mask_obs with header seq: 18 +stamp: + secs: 1733881934 + nsecs: 10175704 +frame_id: '' and grasped_object_mask header seq: 7845 +stamp: + secs: 1733881933 + nsecs: 770703554 +frame_id: "camera_color_optical_frame" +[INFO] [1733881934.204773]: received mask_obs with header seq: 19 +stamp: + secs: 1733881934 + nsecs: 203893184 +frame_id: '' and grasped_object_mask header seq: 7851 +stamp: + secs: 1733881933 + nsecs: 970544338 +frame_id: "camera_color_optical_frame" +[INFO] [1733881934.433582]: received mask_obs with header seq: 20 +stamp: + secs: 1733881934 + nsecs: 432651519 +frame_id: '' and grasped_object_mask header seq: 7857 +stamp: + secs: 1733881934 + nsecs: 170385599 +frame_id: "camera_color_optical_frame" +[INFO] [1733881934.628506]: received mask_obs with header seq: 21 +stamp: + secs: 1733881934 + nsecs: 626872062 +frame_id: '' and grasped_object_mask header seq: 7864 +stamp: + secs: 1733881934 + nsecs: 403534174 +frame_id: "camera_color_optical_frame" +[INFO] [1733881934.826157]: received mask_obs with header seq: 22 +stamp: + secs: 1733881934 + nsecs: 825274705 +frame_id: '' and grasped_object_mask header seq: 7870 +stamp: + secs: 1733881934 + nsecs: 603375435 +frame_id: "camera_color_optical_frame" +[INFO] [1733881935.031275]: received mask_obs with header seq: 23 +stamp: + secs: 1733881935 + nsecs: 29773235 +frame_id: '' and grasped_object_mask header seq: 7876 +stamp: + secs: 1733881934 + nsecs: 803215981 +frame_id: "camera_color_optical_frame" +[INFO] [1733881935.228458]: received mask_obs with header seq: 24 +stamp: + secs: 1733881935 + nsecs: 227554321 +frame_id: '' and grasped_object_mask header seq: 7882 +stamp: + secs: 1733881935 + nsecs: 3057957 +frame_id: "camera_color_optical_frame" +[INFO] [1733881935.422727]: received mask_obs with header seq: 25 +stamp: + secs: 1733881935 + nsecs: 421752691 +frame_id: '' and grasped_object_mask header seq: 7887 +stamp: + secs: 1733881935 + nsecs: 169591427 +frame_id: "camera_color_optical_frame" +[INFO] [1733881935.638195]: received mask_obs with header seq: 26 +stamp: + secs: 1733881935 + nsecs: 637306213 +frame_id: '' and grasped_object_mask header seq: 7894 +stamp: + secs: 1733881935 + nsecs: 402739286 +frame_id: "camera_color_optical_frame" +[INFO] [1733881935.837922]: received mask_obs with header seq: 27 +stamp: + secs: 1733881935 + nsecs: 837003231 +frame_id: '' and grasped_object_mask header seq: 7900 +stamp: + secs: 1733881935 + nsecs: 602580309 +frame_id: "camera_color_optical_frame" +[INFO] [1733881936.034971]: received mask_obs with header seq: 28 +stamp: + secs: 1733881936 + nsecs: 33885240 +frame_id: '' and grasped_object_mask header seq: 7906 +stamp: + secs: 1733881935 + nsecs: 802418470 +frame_id: "camera_color_optical_frame" +[INFO] [1733881936.223721]: received mask_obs with header seq: 29 +stamp: + secs: 1733881936 + nsecs: 222499608 +frame_id: '' and grasped_object_mask header seq: 7912 +stamp: + secs: 1733881936 + nsecs: 2255917 +frame_id: "camera_color_optical_frame" +[INFO] [1733881936.446193]: received mask_obs with header seq: 30 +stamp: + secs: 1733881936 + nsecs: 445185184 +frame_id: '' and grasped_object_mask header seq: 7918 +stamp: + secs: 1733881936 + nsecs: 202094316 +frame_id: "camera_color_optical_frame" +[INFO] [1733881936.632934]: received mask_obs with header seq: 31 +stamp: + secs: 1733881936 + nsecs: 631910800 +frame_id: '' and grasped_object_mask header seq: 7924 +stamp: + secs: 1733881936 + nsecs: 401931524 +frame_id: "camera_color_optical_frame" +[INFO] [1733881936.838232]: received mask_obs with header seq: 32 +stamp: + secs: 1733881936 + nsecs: 837055444 +frame_id: '' and grasped_object_mask header seq: 7930 +stamp: + secs: 1733881936 + nsecs: 601768732 +frame_id: "camera_color_optical_frame" +[INFO] [1733881937.032954]: received mask_obs with header seq: 33 +stamp: + secs: 1733881937 + nsecs: 32030105 +frame_id: '' and grasped_object_mask header seq: 7936 +stamp: + secs: 1733881936 + nsecs: 801601171 +frame_id: "camera_color_optical_frame" +[INFO] [1733881937.227681]: received mask_obs with header seq: 34 +stamp: + secs: 1733881937 + nsecs: 225735902 +frame_id: '' and grasped_object_mask header seq: 7942 +stamp: + secs: 1733881937 + nsecs: 1431465 +frame_id: "camera_color_optical_frame" +[INFO] [1733881937.424884]: received mask_obs with header seq: 35 +stamp: + secs: 1733881937 + nsecs: 423624753 +frame_id: '' and grasped_object_mask header seq: 7948 +stamp: + secs: 1733881937 + nsecs: 201261759 +frame_id: "camera_color_optical_frame" +[INFO] [1733881937.602132]: received mask_obs with header seq: 36 +stamp: + secs: 1733881937 + nsecs: 601197004 +frame_id: '' and grasped_object_mask header seq: 7953 +stamp: + secs: 1733881937 + nsecs: 367787123 +frame_id: "camera_color_optical_frame" +[INFO] [1733881937.819760]: received mask_obs with header seq: 37 +stamp: + secs: 1733881937 + nsecs: 817212343 +frame_id: '' and grasped_object_mask header seq: 7959 +stamp: + secs: 1733881937 + nsecs: 567616940 +frame_id: "camera_color_optical_frame" +[INFO] [1733881938.023777]: received mask_obs with header seq: 38 +stamp: + secs: 1733881938 + nsecs: 22439956 +frame_id: '' and grasped_object_mask header seq: 7965 +stamp: + secs: 1733881937 + nsecs: 767450571 +frame_id: "camera_color_optical_frame" +[INFO] [1733881938.203714]: received mask_obs with header seq: 39 +stamp: + secs: 1733881938 + nsecs: 202681779 +frame_id: '' and grasped_object_mask header seq: 7971 +stamp: + secs: 1733881937 + nsecs: 967287064 +frame_id: "camera_color_optical_frame" +[INFO] [1733881938.403625]: received mask_obs with header seq: 40 +stamp: + secs: 1733881938 + nsecs: 402268409 +frame_id: '' and grasped_object_mask header seq: 7977 +stamp: + secs: 1733881938 + nsecs: 167124510 +frame_id: "camera_color_optical_frame" +[INFO] [1733881938.600104]: received mask_obs with header seq: 41 +stamp: + secs: 1733881938 + nsecs: 598839759 +frame_id: '' and grasped_object_mask header seq: 7983 +stamp: + secs: 1733881938 + nsecs: 366960764 +frame_id: "camera_color_optical_frame" +[INFO] [1733881938.820781]: received mask_obs with header seq: 42 +stamp: + secs: 1733881938 + nsecs: 819800138 +frame_id: '' and grasped_object_mask header seq: 7989 +stamp: + secs: 1733881938 + nsecs: 566796780 +frame_id: "camera_color_optical_frame" +[INFO] [1733881939.014330]: received mask_obs with header seq: 43 +stamp: + secs: 1733881939 + nsecs: 13423442 +frame_id: '' and grasped_object_mask header seq: 7996 +stamp: + secs: 1733881938 + nsecs: 799937010 +frame_id: "camera_color_optical_frame" +[INFO] [1733881939.224791]: received mask_obs with header seq: 44 +stamp: + secs: 1733881939 + nsecs: 223258733 +frame_id: '' and grasped_object_mask header seq: 8001 +stamp: + secs: 1733881938 + nsecs: 966463566 +frame_id: "camera_color_optical_frame" +[INFO] [1733881939.425529]: received mask_obs with header seq: 45 +stamp: + secs: 1733881939 + nsecs: 424565792 +frame_id: '' and grasped_object_mask header seq: 8008 +stamp: + secs: 1733881939 + nsecs: 199602365 +frame_id: "camera_color_optical_frame" +[INFO] [1733881939.619431]: received mask_obs with header seq: 46 +stamp: + secs: 1733881939 + nsecs: 618312597 +frame_id: '' and grasped_object_mask header seq: 8014 +stamp: + secs: 1733881939 + nsecs: 399434566 +frame_id: "camera_color_optical_frame" +[INFO] [1733881939.820555]: received mask_obs with header seq: 47 +stamp: + secs: 1733881939 + nsecs: 819616794 +frame_id: '' and grasped_object_mask header seq: 8020 +stamp: + secs: 1733881939 + nsecs: 599266529 +frame_id: "camera_color_optical_frame" +[INFO] [1733881940.005593]: received mask_obs with header seq: 48 +stamp: + secs: 1733881940 + nsecs: 4656076 +frame_id: '' and grasped_object_mask header seq: 8025 +stamp: + secs: 1733881939 + nsecs: 765801430 +frame_id: "camera_color_optical_frame" +[INFO] [1733881940.204921]: received mask_obs with header seq: 49 +stamp: + secs: 1733881940 + nsecs: 203764915 +frame_id: '' and grasped_object_mask header seq: 8031 +stamp: + secs: 1733881939 + nsecs: 965646029 +frame_id: "camera_color_optical_frame" +[INFO] [1733881940.413592]: received mask_obs with header seq: 50 +stamp: + secs: 1733881940 + nsecs: 412583351 +frame_id: '' and grasped_object_mask header seq: 8037 +stamp: + secs: 1733881940 + nsecs: 165490627 +frame_id: "camera_color_optical_frame" +[INFO] [1733881940.618829]: received mask_obs with header seq: 51 +stamp: + secs: 1733881940 + nsecs: 617256879 +frame_id: '' and grasped_object_mask header seq: 8043 +stamp: + secs: 1733881940 + nsecs: 365335941 +frame_id: "camera_color_optical_frame" +[INFO] [1733881940.824649]: received mask_obs with header seq: 52 +stamp: + secs: 1733881940 + nsecs: 823161363 +frame_id: '' and grasped_object_mask header seq: 8050 +stamp: + secs: 1733881940 + nsecs: 598489761 +frame_id: "camera_color_optical_frame" +[INFO] [1733881941.027861]: received mask_obs with header seq: 53 +stamp: + secs: 1733881941 + nsecs: 26394367 +frame_id: '' and grasped_object_mask header seq: 8056 +stamp: + secs: 1733881940 + nsecs: 798328876 +frame_id: "camera_color_optical_frame" +[INFO] [1733881941.225607]: received mask_obs with header seq: 54 +stamp: + secs: 1733881941 + nsecs: 224196434 +frame_id: '' and grasped_object_mask header seq: 8061 +stamp: + secs: 1733881940 + nsecs: 964860439 +frame_id: "camera_color_optical_frame" +[INFO] [1733881941.403943]: received mask_obs with header seq: 55 +stamp: + secs: 1733881941 + nsecs: 402922868 +frame_id: '' and grasped_object_mask header seq: 8067 +stamp: + secs: 1733881941 + nsecs: 164697409 +frame_id: "camera_color_optical_frame" +[INFO] [1733881941.609296]: received mask_obs with header seq: 56 +stamp: + secs: 1733881941 + nsecs: 608402252 +frame_id: '' and grasped_object_mask header seq: 8073 +stamp: + secs: 1733881941 + nsecs: 364534378 +frame_id: "camera_color_optical_frame" +[INFO] [1733881941.801054]: received mask_obs with header seq: 57 +stamp: + secs: 1733881941 + nsecs: 799741744 +frame_id: '' and grasped_object_mask header seq: 8079 +stamp: + secs: 1733881941 + nsecs: 564371347 +frame_id: "camera_color_optical_frame" +[INFO] [1733881942.001835]: received mask_obs with header seq: 58 +stamp: + secs: 1733881942 + nsecs: 708103 +frame_id: '' and grasped_object_mask header seq: 8085 +stamp: + secs: 1733881941 + nsecs: 764206648 +frame_id: "camera_color_optical_frame" +[INFO] [1733881942.199729]: received mask_obs with header seq: 59 +stamp: + secs: 1733881942 + nsecs: 198785305 +frame_id: '' and grasped_object_mask header seq: 8091 +stamp: + secs: 1733881941 + nsecs: 964039087 +frame_id: "camera_color_optical_frame" +[INFO] [1733881942.426365]: received mask_obs with header seq: 60 +stamp: + secs: 1733881942 + nsecs: 425239324 +frame_id: '' and grasped_object_mask header seq: 8097 +stamp: + secs: 1733881942 + nsecs: 163871288 +frame_id: "camera_color_optical_frame" +[INFO] [1733881942.619282]: received mask_obs with header seq: 61 +stamp: + secs: 1733881942 + nsecs: 617688417 +frame_id: '' and grasped_object_mask header seq: 8104 +stamp: + secs: 1733881942 + nsecs: 397009134 +frame_id: "camera_color_optical_frame" +[INFO] [1733881942.806585]: received mask_obs with header seq: 62 +stamp: + secs: 1733881942 + nsecs: 805705308 +frame_id: '' and grasped_object_mask header seq: 8109 +stamp: + secs: 1733881942 + nsecs: 563535213 +frame_id: "camera_color_optical_frame" +[INFO] [1733881943.003788]: received mask_obs with header seq: 63 +stamp: + secs: 1733881943 + nsecs: 2786159 +frame_id: '' and grasped_object_mask header seq: 8115 +stamp: + secs: 1733881942 + nsecs: 763370991 +frame_id: "camera_color_optical_frame" +[INFO] [1733881943.203472]: received mask_obs with header seq: 64 +stamp: + secs: 1733881943 + nsecs: 202393293 +frame_id: '' and grasped_object_mask header seq: 8121 +stamp: + secs: 1733881942 + nsecs: 963209629 +frame_id: "camera_color_optical_frame" +[INFO] [1733881943.406879]: received mask_obs with header seq: 65 +stamp: + secs: 1733881943 + nsecs: 405942440 +frame_id: '' and grasped_object_mask header seq: 8127 +stamp: + secs: 1733881943 + nsecs: 163049221 +frame_id: "camera_color_optical_frame" +[INFO] [1733881943.602010]: received mask_obs with header seq: 66 +stamp: + secs: 1733881943 + nsecs: 600990533 +frame_id: '' and grasped_object_mask header seq: 8133 +stamp: + secs: 1733881943 + nsecs: 362887859 +frame_id: "camera_color_optical_frame" +[INFO] [1733881943.805436]: received mask_obs with header seq: 67 +stamp: + secs: 1733881943 + nsecs: 804544448 +frame_id: '' and grasped_object_mask header seq: 8139 +stamp: + secs: 1733881943 + nsecs: 562726736 +frame_id: "camera_color_optical_frame" +[INFO] [1733881944.003641]: received mask_obs with header seq: 68 +stamp: + secs: 1733881944 + nsecs: 2423763 +frame_id: '' and grasped_object_mask header seq: 8145 +stamp: + secs: 1733881943 + nsecs: 762568951 +frame_id: "camera_color_optical_frame" +[INFO] [1733881944.201632]: received mask_obs with header seq: 69 +stamp: + secs: 1733881944 + nsecs: 200727224 +frame_id: '' and grasped_object_mask header seq: 8151 +stamp: + secs: 1733881943 + nsecs: 962414980 +frame_id: "camera_color_optical_frame" +[INFO] [1733881944.411565]: received mask_obs with header seq: 70 +stamp: + secs: 1733881944 + nsecs: 410659313 +frame_id: '' and grasped_object_mask header seq: 8157 +stamp: + secs: 1733881944 + nsecs: 162261248 +frame_id: "camera_color_optical_frame" +[INFO] [1733881944.608529]: received mask_obs with header seq: 71 +stamp: + secs: 1733881944 + nsecs: 607629060 +frame_id: '' and grasped_object_mask header seq: 8163 +stamp: + secs: 1733881944 + nsecs: 362109184 +frame_id: "camera_color_optical_frame" +[INFO] [1733881944.799691]: received mask_obs with header seq: 72 +stamp: + secs: 1733881944 + nsecs: 798696279 +frame_id: '' and grasped_object_mask header seq: 8169 +stamp: + secs: 1733881944 + nsecs: 561956167 +frame_id: "camera_color_optical_frame" +[INFO] [1733881945.002218]: received mask_obs with header seq: 73 +stamp: + secs: 1733881945 + nsecs: 1061439 +frame_id: '' and grasped_object_mask header seq: 8175 +stamp: + secs: 1733881944 + nsecs: 761801004 +frame_id: "camera_color_optical_frame" +[INFO] [1733881945.202708]: received mask_obs with header seq: 74 +stamp: + secs: 1733881945 + nsecs: 201566219 +frame_id: '' and grasped_object_mask header seq: 8181 +stamp: + secs: 1733881944 + nsecs: 961643934 +frame_id: "camera_color_optical_frame" +[INFO] [1733881945.408593]: received mask_obs with header seq: 75 +stamp: + secs: 1733881945 + nsecs: 407604694 +frame_id: '' and grasped_object_mask header seq: 8187 +stamp: + secs: 1733881945 + nsecs: 161487103 +frame_id: "camera_color_optical_frame" +[INFO] [1733881945.601663]: received mask_obs with header seq: 76 +stamp: + secs: 1733881945 + nsecs: 600501298 +frame_id: '' and grasped_object_mask header seq: 8193 +stamp: + secs: 1733881945 + nsecs: 361330271 +frame_id: "camera_color_optical_frame" +[INFO] [1733881945.808530]: received mask_obs with header seq: 77 +stamp: + secs: 1733881945 + nsecs: 807325363 +frame_id: '' and grasped_object_mask header seq: 8199 +stamp: + secs: 1733881945 + nsecs: 561174870 +frame_id: "camera_color_optical_frame" +[INFO] [1733881946.008040]: received mask_obs with header seq: 78 +stamp: + secs: 1733881946 + nsecs: 7133245 +frame_id: '' and grasped_object_mask header seq: 8205 +stamp: + secs: 1733881945 + nsecs: 761015177 +frame_id: "camera_color_optical_frame" +[INFO] [1733881946.201357]: received mask_obs with header seq: 79 +stamp: + secs: 1733881946 + nsecs: 200472593 +frame_id: '' and grasped_object_mask header seq: 8211 +stamp: + secs: 1733881945 + nsecs: 960851908 +frame_id: "camera_color_optical_frame" +[INFO] [1733881946.401407]: received mask_obs with header seq: 80 +stamp: + secs: 1733881946 + nsecs: 400375843 +frame_id: '' and grasped_object_mask header seq: 8217 +stamp: + secs: 1733881946 + nsecs: 160688162 +frame_id: "camera_color_optical_frame" +[INFO] [1733881946.607615]: received mask_obs with header seq: 81 +stamp: + secs: 1733881946 + nsecs: 606504678 +frame_id: '' and grasped_object_mask header seq: 8223 +stamp: + secs: 1733881946 + nsecs: 360524893 +frame_id: "camera_color_optical_frame" +[INFO] [1733881946.810744]: received mask_obs with header seq: 82 +stamp: + secs: 1733881946 + nsecs: 809859037 +frame_id: '' and grasped_object_mask header seq: 8229 +stamp: + secs: 1733881946 + nsecs: 560361385 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.001563]: received mask_obs with header seq: 83 +stamp: + secs: 1733881947 + nsecs: 424861 +frame_id: '' and grasped_object_mask header seq: 8235 +stamp: + secs: 1733881946 + nsecs: 760201216 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.203573]: received mask_obs with header seq: 84 +stamp: + secs: 1733881947 + nsecs: 202486991 +frame_id: '' and grasped_object_mask header seq: 8241 +stamp: + secs: 1733881946 + nsecs: 960042477 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.405680]: received mask_obs with header seq: 85 +stamp: + secs: 1733881947 + nsecs: 404779672 +frame_id: '' and grasped_object_mask header seq: 8247 +stamp: + secs: 1733881947 + nsecs: 159883976 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.599595]: received mask_obs with header seq: 86 +stamp: + secs: 1733881947 + nsecs: 598672866 +frame_id: '' and grasped_object_mask header seq: 8253 +stamp: + secs: 1733881947 + nsecs: 359725475 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.800599]: received mask_obs with header seq: 87 +stamp: + secs: 1733881947 + nsecs: 799697399 +frame_id: '' and grasped_object_mask header seq: 8259 +stamp: + secs: 1733881947 + nsecs: 559567451 +frame_id: "camera_color_optical_frame" +[INFO] [1733881947.999780]: received mask_obs with header seq: 88 +stamp: + secs: 1733881947 + nsecs: 998712778 +frame_id: '' and grasped_object_mask header seq: 8265 +stamp: + secs: 1733881947 + nsecs: 759409428 +frame_id: "camera_color_optical_frame" +[INFO] [1733881948.207681]: received mask_obs with header seq: 89 +stamp: + secs: 1733881948 + nsecs: 206808567 +frame_id: '' and grasped_object_mask header seq: 8271 +stamp: + secs: 1733881947 + nsecs: 959252834 +frame_id: "camera_color_optical_frame" +[INFO] [1733881948.404581]: received mask_obs with header seq: 90 +stamp: + secs: 1733881948 + nsecs: 403736352 +frame_id: '' and grasped_object_mask header seq: 8277 +stamp: + secs: 1733881948 + nsecs: 159095764 +frame_id: "camera_color_optical_frame" +[INFO] [1733881948.607006]: received mask_obs with header seq: 91 +stamp: + secs: 1733881948 + nsecs: 605962514 +frame_id: '' and grasped_object_mask header seq: 8283 +stamp: + secs: 1733881948 + nsecs: 358938456 +frame_id: "camera_color_optical_frame" +[INFO] [1733881948.826154]: received mask_obs with header seq: 92 +stamp: + secs: 1733881948 + nsecs: 824640035 +frame_id: '' and grasped_object_mask header seq: 8289 +stamp: + secs: 1733881948 + nsecs: 558781624 +frame_id: "camera_color_optical_frame" +[INFO] [1733881949.042130]: received mask_obs with header seq: 93 +stamp: + secs: 1733881949 + nsecs: 41005611 +frame_id: '' and grasped_object_mask header seq: 8296 +stamp: + secs: 1733881948 + nsecs: 791931868 +frame_id: "camera_color_optical_frame" +[INFO] [1733881949.253016]: received mask_obs with header seq: 94 +stamp: + secs: 1733881949 + nsecs: 251410484 +frame_id: '' and grasped_object_mask header seq: 8302 +stamp: + secs: 1733881948 + nsecs: 991774082 +frame_id: "camera_color_optical_frame" +[INFO] [1733881949.451710]: received mask_obs with header seq: 95 +stamp: + secs: 1733881949 + nsecs: 449931383 +frame_id: '' and grasped_object_mask header seq: 8308 +stamp: + secs: 1733881949 + nsecs: 191616774 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 110894429 for nearest: 1733881949502353656 - target: 1733881949391459227 at idx 249 +[INFO] [1733881949.644919]: received mask_obs with header seq: 96 +stamp: + secs: 1733881949 + nsecs: 644002914 +frame_id: '' and grasped_object_mask header seq: 8314 +stamp: + secs: 1733881949 + nsecs: 391459227 +frame_id: "camera_color_optical_frame" +[INFO] [1733881949.839658]: received mask_obs with header seq: 97 +stamp: + secs: 1733881949 + nsecs: 838388442 +frame_id: '' and grasped_object_mask header seq: 8320 +stamp: + secs: 1733881949 + nsecs: 591301680 +frame_id: "camera_color_optical_frame" +[INFO] [1733881950.026284]: received mask_obs with header seq: 98 +stamp: + secs: 1733881950 + nsecs: 24919033 +frame_id: '' and grasped_object_mask header seq: 8326 +stamp: + secs: 1733881949 + nsecs: 791146994 +frame_id: "camera_color_optical_frame" +[INFO] [1733881950.241674]: received mask_obs with header seq: 99 +stamp: + secs: 1733881950 + nsecs: 240784645 +frame_id: '' and grasped_object_mask header seq: 8332 +stamp: + secs: 1733881949 + nsecs: 990992785 +frame_id: "camera_color_optical_frame" +[INFO] [1733881950.441060]: received mask_obs with header seq: 100 +stamp: + secs: 1733881950 + nsecs: 439967393 +frame_id: '' and grasped_object_mask header seq: 8338 +stamp: + secs: 1733881950 + nsecs: 190838814 +frame_id: "camera_color_optical_frame" +[INFO] [1733881950.630870]: received mask_obs with header seq: 101 +stamp: + secs: 1733881950 + nsecs: 629913806 +frame_id: '' and grasped_object_mask header seq: 8344 +stamp: + secs: 1733881950 + nsecs: 390685320 +frame_id: "camera_color_optical_frame" +[INFO] [1733881950.832320]: received mask_obs with header seq: 102 +stamp: + secs: 1733881950 + nsecs: 831316471 +frame_id: '' and grasped_object_mask header seq: 8350 +stamp: + secs: 1733881950 + nsecs: 590532303 +frame_id: "camera_color_optical_frame" +[INFO] [1733881951.028132]: received mask_obs with header seq: 103 +stamp: + secs: 1733881951 + nsecs: 27282714 +frame_id: '' and grasped_object_mask header seq: 8356 +stamp: + secs: 1733881950 + nsecs: 790374041 +frame_id: "camera_color_optical_frame" +[INFO] [1733881951.215708]: received mask_obs with header seq: 104 +stamp: + secs: 1733881951 + nsecs: 214742898 +frame_id: '' and grasped_object_mask header seq: 8362 +stamp: + secs: 1733881950 + nsecs: 990212679 +frame_id: "camera_color_optical_frame" +[INFO] [1733881951.419554]: received mask_obs with header seq: 105 +stamp: + secs: 1733881951 + nsecs: 418620824 +frame_id: '' and grasped_object_mask header seq: 8368 +stamp: + secs: 1733881951 + nsecs: 190052032 +frame_id: "camera_color_optical_frame" +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1733881951.619548]: received mask_obs with header seq: 106 +stamp: + secs: 1733881951 + nsecs: 603054285 +frame_id: '' and grasped_object_mask header seq: 8373 +stamp: + secs: 1733881951 + nsecs: 356583834 +frame_id: "camera_color_optical_frame" +[INFO] [1733881951.832582]: received mask_obs with header seq: 107 +stamp: + secs: 1733881951 + nsecs: 831149339 +frame_id: '' and grasped_object_mask header seq: 8380 +stamp: + secs: 1733881951 + nsecs: 589729309 +frame_id: "camera_color_optical_frame" +[INFO] [1733881952.050082]: received mask_obs with header seq: 108 +stamp: + secs: 1733881952 + nsecs: 46733140 +frame_id: '' and grasped_object_mask header seq: 8386 +stamp: + secs: 1733881951 + nsecs: 789563179 +frame_id: "camera_color_optical_frame" +[INFO] [1733881952.244104]: received mask_obs with header seq: 109 +stamp: + secs: 1733881952 + nsecs: 243089437 +frame_id: '' and grasped_object_mask header seq: 8393 +stamp: + secs: 1733881952 + nsecs: 22698879 +frame_id: "camera_color_optical_frame" +Exception in thread Thread-62: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/FISH/video.py", line 78, in ros_record_cb + self.process.stdin.write(frame.tobytes()) +BrokenPipeError: [Errno 32] Broken pipe +program exiting gracefully +Exception in thread Thread-71: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 585, in ros_step_cb + mask_header, mask_obs_msg = self.wait_for_exact_match(frame_metadata['rgb_header'].stamp, self.mask_obs_queue) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 636, in wait_for_exact_match + assert queue_return is not None, f"Exact match not found in {timeout} seconds" +AssertionError: Exact match not found in 5 seconds +proceeding to start episode! +[INFO] [1733881964.331730]: Saving video... +[INFO] [1733881964.332162]: Video saved! \ No newline at end of file diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt b/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json b/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2af8e1e10fe49020010b88fd9d56993638f88156 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T01:51:30.761221", + "startedAt": "2024-12-11T01:51:30.302896", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4124.7015, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.948, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.997, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 364.1780319213867 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json b/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..06910cf736887685d5132ab83697413509866fa7 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/video": {"_type": "video-file", "sha256": "b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10", "size": 942048, "path": "media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4"}, "global_step": 109, "_timestamp": 1733881964.3609016, "_runtime": 74.04860258102417, "_step": 0, "_wandb": {"runtime": 73}} \ No newline at end of file diff --git a/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log b/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..778b889c4c425bdaf3eb6b38f70268482f9cc6a8 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log @@ -0,0 +1,351 @@ +2024-12-10 20:51:30,311 INFO StreamThr :597014 [internal.py:wandb_internal():85] W&B internal server running at pid: 597014, started at: 2024-12-10 20:51:30.311157 +2024-12-10 20:51:30,312 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:51:30,325 INFO WriterThread:597014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb +2024-12-10 20:51:30,325 DEBUG SenderThread:597014 [sender.py:send():379] send: header +2024-12-10 20:51:30,336 DEBUG SenderThread:597014 [sender.py:send():379] send: run +2024-12-10 20:51:30,645 INFO SenderThread:597014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files +2024-12-10 20:51:30,645 INFO SenderThread:597014 [sender.py:_start_run_threads():1188] run started: svysfc56 with start time 1733881890.312299 +2024-12-10 20:51:30,656 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:51:30,656 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:51:30,743 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:51:30,748 DEBUG HandlerThread:597014 [system_info.py:__init__():26] System info init +2024-12-10 20:51:30,748 DEBUG HandlerThread:597014 [system_info.py:__init__():41] System info init done +2024-12-10 20:51:30,748 INFO HandlerThread:597014 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:51:30,748 INFO SystemMonitor:597014 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:51:30,749 INFO HandlerThread:597014 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:51:30,749 INFO SystemMonitor:597014 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:51:30,750 INFO SystemMonitor:597014 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:51:30,750 INFO SystemMonitor:597014 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:51:30,751 INFO SystemMonitor:597014 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:51:30,751 INFO SystemMonitor:597014 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:51:30,761 DEBUG HandlerThread:597014 [system_info.py:probe():152] Probing system +2024-12-10 20:51:30,762 DEBUG HandlerThread:597014 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_info.py:probe():200] Probing system done +2024-12-10 20:51:30,767 DEBUG HandlerThread:597014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:51:30.761221', 'startedAt': '2024-12-11T01:51:30.302896', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.7015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 364.1780319213867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:51:30,768 INFO HandlerThread:597014 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:51:30,768 INFO HandlerThread:597014 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:51:30,768 DEBUG HandlerThread:597014 [system_info.py:_save_code():44] Saving code +2024-12-10 20:51:30,769 DEBUG HandlerThread:597014 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:51:30,769 DEBUG HandlerThread:597014 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:51:30,832 DEBUG HandlerThread:597014 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:51:30,832 INFO HandlerThread:597014 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:51:30,836 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:51:30,836 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:51:31,124 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/tiz5ae9w-diff.patch +2024-12-10 20:51:31,126 INFO wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/p6o167mp-wandb-metadata.json +2024-12-10 20:51:31,143 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:51:31,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:51:31,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:51:31,145 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:51:31,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:31,147 INFO HandlerThread:597014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb" + save: true +} +control { + relay_id: "42f7013eb6ef4a01b21dd778abc8c686" +} +_info { + stream_id: "svysfc56" +} + +2024-12-10 20:51:31,147 DEBUG HandlerThread:597014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:51:31,173 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/9ln43j5e-code/FISH/eval_robot.py +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: telemetry +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: tbrecord +2024-12-10 20:51:31,218 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:51:31,218 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 with policy live +2024-12-10 20:51:31,415 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/08emxf8k-events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH +2024-12-10 20:51:31,647 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code +2024-12-10 20:51:32,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:33,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:33,646 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:34,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:35,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:36,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:36,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:37,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:38,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:39,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:40,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:41,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:41,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:42,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:43,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:44,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:45,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:46,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:51:46,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:51:46,186 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:46,300 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:46,908 DEBUG SenderThread:597014 [sender.py:send():379] send: config +2024-12-10 20:51:47,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:48,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:49,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:49,651 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:50,146 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:51,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:51,335 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:52,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:53,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:53,652 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:54,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:55,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:56,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:57,145 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:51:57,147 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:57,654 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:51:58,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:51:59,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:00,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:01,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:01,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:01,189 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:02,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:02,300 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:02,655 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml +2024-12-10 20:52:03,148 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:04,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:05,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:06,182 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:07,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:07,657 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:08,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:08,194 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:09,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:09,657 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:10,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:11,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:11,658 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:12,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:13,149 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:13,219 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:13,659 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:14,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:15,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:15,659 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:16,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:16,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:16,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:17,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:17,660 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:18,150 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:18,404 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:19,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:19,661 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:20,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:21,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:21,661 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:22,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:23,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:23,407 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:23,662 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:24,152 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:25,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:25,662 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:26,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:27,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:27,663 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:28,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:28,607 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:29,153 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:29,664 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:30,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:30,752 DEBUG SystemMonitor:597014 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:52:30,752 DEBUG SenderThread:597014 [sender.py:send():379] send: stats +2024-12-10 20:52:31,144 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:52:31,144 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:52:31,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:31,666 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:32,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:33,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:33,667 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:33,908 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:34,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:34,667 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:35,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:35,668 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:36,154 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:37,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:38,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:38,668 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:39,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:39,371 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:40,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:41,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:42,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:43,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:44,155 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:44,374 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send():379] send: files +2024-12-10 20:52:44,375 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 with policy now +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send():379] send: history +2024-12-10 20:52:44,375 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:44,375 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:44,377 DEBUG SenderThread:597014 [sender.py:send():379] send: exit +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():588] handling runtime: 73 +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:44,377 INFO SenderThread:597014 [sender.py:send_exit():594] send defer +2024-12-10 20:52:44,379 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,379 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:52:44,380 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:52:44,380 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:52:44,380 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:52:44,380 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:52:44,380 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:52:44,380 INFO HandlerThread:597014 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:52:44,381 DEBUG SystemMonitor:597014 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:52:44,381 DEBUG SystemMonitor:597014 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:52:44,381 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:52:44,382 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:52:44,439 INFO HandlerThread:597014 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send():379] send: stats +2024-12-10 20:52:44,440 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,440 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:52:44,440 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:52:44,440 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:52:44,441 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:44,441 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:52:44,670 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos +2024-12-10 20:52:44,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media +2024-12-10 20:52:44,696 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/6w704qfx-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:45,030 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/kk0w0e5k-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:45,671 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:51,156 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,157 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:52:51,157 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:51,157 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,157 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:52:51,157 DEBUG SenderThread:597014 [sender.py:send():379] send: summary +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:52:51,158 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,158 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:52:51,158 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:52:51,158 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:52:51,158 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,158 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:52:51,159 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,159 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:52:51,375 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:51,673 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:51,921 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:52:51,922 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:51,922 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,922 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:52:51,922 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:51,922 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:52:51,922 INFO SenderThread:597014 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:52:51,924 INFO SenderThread:597014 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:52:51,942 INFO SenderThread:597014 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:52:51,951 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:52:51,952 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:51,952 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:52:51,952 DEBUG SenderThread:597014 [sender.py:send():379] send: artifact +2024-12-10 20:52:52,376 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:52,626 INFO wandb-upload_0:597014 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmptoffctew +2024-12-10 20:52:52,673 INFO Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:52,858 INFO wandb-upload_2:597014 [upload_job.py:push():88] Uploaded file /tmp/tmpemx2d4yl/wandb-job.json +2024-12-10 20:52:52,880 INFO wandb-upload_1:597014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpc2y7tffk +2024-12-10 20:52:53,418 INFO SenderThread:597014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM1MTcyNjQ1OA==', 'versionIndex': 335}}} +2024-12-10 20:52:53,418 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:53,418 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:52:53,418 INFO SenderThread:597014 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml config.yaml +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json wandb-summary.json +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt requirements.txt +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 +2024-12-10 20:52:53,674 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log output.log +2024-12-10 20:52:53,676 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch diff.patch +2024-12-10 20:52:53,680 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:52:53,680 INFO SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 +2024-12-10 20:52:53,680 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:52:53,680 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:53,681 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:53,681 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:52:53,681 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:53,681 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:52:53,681 INFO SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:52:53,872 INFO wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt +2024-12-10 20:52:53,888 INFO wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml +2024-12-10 20:52:53,899 INFO wandb-upload_3:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json +2024-12-10 20:52:53,910 INFO wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log +2024-12-10 20:52:54,110 INFO Thread-11 (_thread_body):597014 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:52:54,110 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,111 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:52:54,111 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,111 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:52:54,111 INFO SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:52:54,111 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:52:54,111 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,111 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:52:54,112 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,112 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:52:54,112 INFO SenderThread:597014 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:52:54,317 INFO SenderThread:597014 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:52:54,317 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:52:54,318 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,318 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:52:54,318 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,318 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:52:54,318 INFO SenderThread:597014 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:52:54,318 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:52:54,318 INFO HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send():379] send: final +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send():379] send: footer +2024-12-10 20:52:54,319 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: defer +2024-12-10 20:52:54,319 INFO SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:52:54,319 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:54,320 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:54,320 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:52:54,320 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:52:54,322 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:52:54,378 DEBUG SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:52:54,378 INFO MainThread:597014 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:52:54,378 INFO MainThread:597014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:52:54,379 INFO MainThread:597014 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:52:54,379 DEBUG HandlerThread:597014 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:52:54,379 INFO HandlerThread:597014 [handler.py:finish():882] shutting down handler +2024-12-10 20:52:55,322 INFO WriterThread:597014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb +2024-12-10 20:52:55,378 INFO SenderThread:597014 [sender.py:finish():1615] shutting down sender +2024-12-10 20:52:55,378 INFO SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:52:55,378 INFO SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher diff --git a/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log b/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..10052a0fe4a55bedce5410671ea78339d38a1ac4 --- /dev/null +++ b/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Configure stats pid to 596892 +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/settings +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:51:30,303 INFO MainThread:596892 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': True}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():619] starting backend +2024-12-10 20:51:30,304 INFO MainThread:596892 [wandb_init.py:init():623] setting up manager +2024-12-10 20:51:30,310 INFO MainThread:596892 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:51:30,312 INFO MainThread:596892 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:51:30,324 INFO MainThread:596892 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:51:30,334 INFO MainThread:596892 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:51:30,656 INFO MainThread:596892 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:51:30,737 INFO MainThread:596892 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:51:30,737 INFO MainThread:596892 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:51:31,144 INFO MainThread:596892 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:51:31,145 INFO MainThread:596892 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:51:31,147 INFO MainThread:596892 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:51:31,147 INFO MainThread:596892 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb, True +2024-12-10 20:51:46,908 INFO MainThread:596892 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:52:55,379 WARNING MsgRouterThr:596892 [router.py:message_loop():77] message_loop has been closed diff --git a/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb b/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb new file mode 100644 index 0000000000000000000000000000000000000000..66f97705cc945739b0944d47267034a5288afc89 Binary files /dev/null and b/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb differ diff --git a/205307/.hydra/config.yaml b/205307/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/205307/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205307/.hydra/hydra.yaml b/205307/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51b1a658143d742cd349657a8fa6b1c14e64a739 --- /dev/null +++ b/205307/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205307 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205307/eval_policy.log b/205307/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..dd279969bf7548dee6fba468dffb6151985c0ce3 --- /dev/null +++ b/205307/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-10 20:53:07,380][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:53:07,383][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-10 20:53:08,679][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/205324/.hydra/config.yaml b/205324/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/205324/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205324/.hydra/hydra.yaml b/205324/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..feeef029b2e3b59ba61fc44f4188466b8363ecfb --- /dev/null +++ b/205324/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205324/.hydra/overrides.yaml b/205324/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205324/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205324/episode_rosbags/aligned_depth_to_color_K.npy b/205324/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205324/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205324/episode_rosbags/cam_tf_world.npy b/205324/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/205324/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/205324/episode_rosbags/color_K.npy b/205324/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205324/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205324/episode_rosbags/depth_K.npy b/205324/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/205324/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/205324/eval_robot.log b/205324/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..52d9c4ac5ef19989d508d99b16adb51db9d8659e --- /dev/null +++ b/205324/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 20:53:24,743][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:53:24,746][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/205324/wandb/debug-internal.log b/205324/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a3b8f02e7c65c420ba1a8d52afd6e7d2ba07e98a --- /dev/null +++ b/205324/wandb/debug-internal.log @@ -0,0 +1,327 @@ +2024-12-10 20:53:30,850 INFO StreamThr :598006 [internal.py:wandb_internal():85] W&B internal server running at pid: 598006, started at: 2024-12-10 20:53:30.850039 +2024-12-10 20:53:30,851 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:53:30,863 INFO WriterThread:598006 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb +2024-12-10 20:53:30,863 DEBUG SenderThread:598006 [sender.py:send():379] send: header +2024-12-10 20:53:30,875 DEBUG SenderThread:598006 [sender.py:send():379] send: run +2024-12-10 20:53:31,147 INFO SenderThread:598006 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files +2024-12-10 20:53:31,147 INFO SenderThread:598006 [sender.py:_start_run_threads():1188] run started: 52nddla7 with start time 1733882010.850819 +2024-12-10 20:53:31,161 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:53:31,161 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:53:31,206 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:53:31,211 DEBUG HandlerThread:598006 [system_info.py:__init__():26] System info init +2024-12-10 20:53:31,211 DEBUG HandlerThread:598006 [system_info.py:__init__():41] System info init done +2024-12-10 20:53:31,211 INFO HandlerThread:598006 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:53:31,211 INFO SystemMonitor:598006 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:53:31,212 INFO HandlerThread:598006 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:53:31,212 INFO SystemMonitor:598006 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:53:31,212 INFO SystemMonitor:598006 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:53:31,213 INFO SystemMonitor:598006 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:53:31,213 INFO SystemMonitor:598006 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:53:31,214 INFO SystemMonitor:598006 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:53:31,224 DEBUG HandlerThread:598006 [system_info.py:probe():152] Probing system +2024-12-10 20:53:31,226 DEBUG HandlerThread:598006 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:probe():200] Probing system done +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:53:31.224621', 'startedAt': '2024-12-11T01:53:30.842354', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.2335090637207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:53:31,231 INFO HandlerThread:598006 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:53:31,231 INFO HandlerThread:598006 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:_save_code():44] Saving code +2024-12-10 20:53:31,233 DEBUG HandlerThread:598006 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:53:31,233 DEBUG HandlerThread:598006 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:53:31,305 DEBUG HandlerThread:598006 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:53:31,306 INFO HandlerThread:598006 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:53:31,309 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:53:31,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:53:31,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:53:31,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:53:31,571 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/oqlixunj-wandb-metadata.json +2024-12-10 20:53:31,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:31,572 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:53:31,573 INFO HandlerThread:598006 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb" + save: true +} +control { + relay_id: "5d813b8bb5a547c8896a7e03615dca84" +} +_info { + stream_id: "52nddla7" +} + +2024-12-10 20:53:31,574 DEBUG HandlerThread:598006 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:53:31,605 INFO wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/6yv6eek3-code/FISH/eval_robot.py +2024-12-10 20:53:31,648 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/w33l6u66-diff.patch +2024-12-10 20:53:31,702 DEBUG SenderThread:598006 [sender.py:send():379] send: telemetry +2024-12-10 20:53:31,702 DEBUG SenderThread:598006 [sender.py:send():379] send: tbrecord +2024-12-10 20:53:31,703 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:53:31,703 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 with policy live +2024-12-10 20:53:31,907 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/uat8jiac-events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code +2024-12-10 20:53:32,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:33,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:34,148 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:34,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:35,547 DEBUG SenderThread:598006 [sender.py:send():379] send: config +2024-12-10 20:53:35,590 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:36,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:36,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:37,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:38,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:38,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:39,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:40,150 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:40,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:41,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:42,082 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:42,150 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:42,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:43,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:44,151 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:44,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:45,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:46,151 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:46,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:53:46,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:53:46,614 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:47,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:47,640 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:48,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:49,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:50,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:51,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:52,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:53,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:53,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:54,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:55,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:56,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:57,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:58,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:58,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:59,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:00,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:01,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:01,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:01,613 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:02,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:03,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:03,722 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:04,156 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml +2024-12-10 20:54:04,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:05,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:06,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:07,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:08,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:09,550 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:09,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:10,158 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:10,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:11,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:12,159 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:12,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:13,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:14,159 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:14,567 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:14,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:15,160 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:15,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:16,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:16,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:16,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:17,160 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:17,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:18,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:19,161 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:19,567 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:19,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:20,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:21,161 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:21,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:22,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:23,162 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:23,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:24,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:24,607 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:25,162 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:25,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:26,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:27,163 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:27,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:28,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:29,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:29,254 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:54:29,254 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 with policy now +2024-12-10 20:54:29,256 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:54:29,257 DEBUG SenderThread:598006 [sender.py:send():379] send: history +2024-12-10 20:54:29,257 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:54:29,257 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:29,258 DEBUG SenderThread:598006 [sender.py:send():379] send: exit +2024-12-10 20:54:29,258 INFO SenderThread:598006 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_exit():588] handling runtime: 58 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_exit():594] send defer +2024-12-10 20:54:29,259 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,259 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:54:29,259 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:54:29,259 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,259 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:54:29,260 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,260 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:54:29,260 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:54:29,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,260 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:54:29,260 INFO HandlerThread:598006 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:54:29,269 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:54:29,269 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:54:29,319 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:54:29,319 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:54:29,320 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:54:29,320 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,320 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:54:29,320 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:54:29,320 DEBUG SenderThread:598006 [sender.py:send():379] send: stats +2024-12-10 20:54:29,320 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,320 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:54:29,321 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,321 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:54:29,321 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:54:29,321 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,321 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:54:29,518 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/0kw42edo-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media +2024-12-10 20:54:30,523 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/ks8s91l3-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,581 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,581 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:54:35,581 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:54:35,581 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:35,582 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,582 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:54:35,582 DEBUG SenderThread:598006 [sender.py:send():379] send: summary +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:35,582 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,583 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:54:35,583 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,583 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:54:35,583 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:54:36,166 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:36,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:37,110 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:54:37,110 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:37,110 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:37,110 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:54:37,111 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:37,111 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:54:37,111 INFO SenderThread:598006 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:54:37,113 INFO SenderThread:598006 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:54:37,127 INFO SenderThread:598006 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:54:37,136 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:54:37,136 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:37,136 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:54:37,137 DEBUG SenderThread:598006 [sender.py:send():379] send: artifact +2024-12-10 20:54:37,166 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:37,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:37,780 INFO wandb-upload_1:598006 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_p_p260o +2024-12-10 20:54:37,783 INFO wandb-upload_0:598006 [upload_job.py:push():85] Skipped uploading /tmp/tmppldao70p/wandb-job.json +2024-12-10 20:54:37,954 INFO wandb-upload_2:598006 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4q7vuxho +2024-12-10 20:54:38,567 INFO SenderThread:598006 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'versionIndex': 336}}} +2024-12-10 20:54:38,567 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:38,567 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:54:38,567 INFO SenderThread:598006 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:54:39,167 INFO SenderThread:598006 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml config.yaml +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json wandb-summary.json +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt requirements.txt +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log output.log +2024-12-10 20:54:39,173 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch diff.patch +2024-12-10 20:54:39,173 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:54:39,174 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:39,175 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:54:39,175 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,175 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,175 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:54:39,175 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,175 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:54:39,175 INFO SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:54:39,261 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,261 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,363 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:39,397 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt +2024-12-10 20:54:39,416 INFO wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml +2024-12-10 20:54:39,447 INFO wandb-upload_3:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:39,647 INFO Thread-11 (_thread_body):598006 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:54:39,647 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,647 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:54:39,648 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,648 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:54:39,648 INFO SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:54:39,648 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:54:39,648 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,648 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:54:39,648 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,649 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:54:39,649 INFO SenderThread:598006 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:54:39,709 INFO SenderThread:598006 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:54:39,709 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,709 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:54:39,709 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:54:39,709 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,709 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send():379] send: final +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send():379] send: footer +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,710 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,711 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,711 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:54:39,712 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:54:39,713 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:54:39,713 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:39,793 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:54:39,794 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:54:39,794 INFO HandlerThread:598006 [handler.py:finish():882] shutting down handler +2024-12-10 20:54:40,712 INFO WriterThread:598006 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb +2024-12-10 20:54:40,794 INFO SenderThread:598006 [sender.py:finish():1615] shutting down sender +2024-12-10 20:54:40,794 INFO SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:54:40,794 INFO SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher diff --git a/205324/wandb/debug.log b/205324/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4ab29e48de854cde31d3ccafd386e4305f1235a5 --- /dev/null +++ b/205324/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Configure stats pid to 597884 +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/settings +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():619] starting backend +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():623] setting up manager +2024-12-10 20:53:30,849 INFO MainThread:597884 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:53:30,850 INFO MainThread:597884 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:53:30,862 INFO MainThread:597884 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:53:30,871 INFO MainThread:597884 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:53:31,160 INFO MainThread:597884 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:53:31,200 INFO MainThread:597884 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:53:31,201 INFO MainThread:597884 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:53:31,573 INFO MainThread:597884 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:53:31,573 INFO MainThread:597884 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb, True +2024-12-10 20:53:35,546 INFO MainThread:597884 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:54:40,795 WARNING MsgRouterThr:597884 [router.py:message_loop():77] message_loop has been closed diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py b/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml b/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..546596c3b6e712b81f9c16aa77b152dba2fc1a98 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733882010 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch b/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..ffeda89f47c1d6b36e048eac4bf0463d100c7103 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch @@ -0,0 +1,34 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..aa52de5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos +@@ -151,7 +156,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history + # wandb_run_id: '250380_0' # RGBD+mask+contact+act history + # wandb_run_id: '250370_0' # RGBD+contact+act history + +-true_action_history: true ++true_action_history: false + wandb_notes: null + + checkpoint_epoch: 12000 # for depth + all contact diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/output.log b/205324/wandb/run-20241210_205330-52nddla7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..78d3798b29b374680a37056220ecf688b380cb22 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/output.log @@ -0,0 +1,461 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733882015.547602]: resetting environment +[INFO] [1733882015.555685]: cleared current plan +[INFO] [1733882015.557298]: moving to home +[INFO] [1733882018.360980]: reached home +[INFO] [1733882018.362525]: reset action history +[INFO] [1733882021.071264]: environment reset +Starting episode 0 +[INFO] [1733882021.073619]: resetting environment +[INFO] [1733882021.078303]: cleared current plan +[INFO] [1733882021.079103]: moving to home +[INFO] [1733882022.081897]: reached home +[INFO] [1733882022.083956]: reset action history +[INFO] [1733882024.791365]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733882047.498704]: resetting environment +[INFO] [1733882047.505692]: cleared current plan +[INFO] [1733882047.506996]: moving to home +[INFO] [1733882048.509273]: reached home +[INFO] [1733882048.510630]: reset action history +[INFO] [1733882051.218254]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/episode_rosbags/episode_0_2024-12-10-20-54-11.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-597884.log +started roslaunch server http://158.130.50.37:45489/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [598110] +process[print_text-2]: started with pid [598134] +process[rosbag_record-3]: started with pid [598135] +started bagging! +[INFO] [1733882051.973240]: received mask_obs with header seq: 110 +stamp: + secs: 1733882051 + nsecs: 972197771 +frame_id: '' and grasped_object_mask header seq: 11387 +stamp: + secs: 1733882051 + nsecs: 753522158 +frame_id: "camera_color_optical_frame" +[INFO] [1733882052.158201]: received mask_obs with header seq: 111 +stamp: + secs: 1733882052 + nsecs: 157648563 +frame_id: '' and grasped_object_mask header seq: 11393 +stamp: + secs: 1733882051 + nsecs: 953104734 +frame_id: "camera_color_optical_frame" +Exception in thread Thread-71: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 562, in ros_step_cb + action_history, plan_headers_list = self.arm.get_action_history( + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 574, in get_action_history + action_history, plan_headers_list = self.get_hallucinated_action_history(prev_plan_timestamp, current_timestamp, history_length) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 599, in get_hallucinated_action_history + assert action_plan_return is not None, f'No plan found after {timeout.to_sec()} seconds' +AssertionError: No plan found after 1.0 seconds +[WARN] [1733882053.167214]: Plan exhausted +[WARN] [1733882053.207056]: Plan exhausted +[WARN] [1733882053.247010]: Plan exhausted +[WARN] [1733882053.287040]: Plan exhausted +[WARN] [1733882053.327405]: Plan exhausted +[WARN] [1733882053.367146]: Plan exhausted +[WARN] [1733882053.407281]: Plan exhausted +[WARN] [1733882053.447035]: Plan exhausted +[WARN] [1733882053.487113]: Plan exhausted +[WARN] [1733882053.527055]: Plan exhausted +[WARN] [1733882053.567043]: Plan exhausted +[WARN] [1733882053.607055]: Plan exhausted +[WARN] [1733882053.647050]: Plan exhausted +[WARN] [1733882053.687051]: Plan exhausted +[WARN] [1733882053.727048]: Plan exhausted +[WARN] [1733882053.767034]: Plan exhausted +[WARN] [1733882053.807052]: Plan exhausted +[WARN] [1733882053.847048]: Plan exhausted +[WARN] [1733882053.887264]: Plan exhausted +[WARN] [1733882053.927126]: Plan exhausted +[WARN] [1733882053.967045]: Plan exhausted +[WARN] [1733882054.007052]: Plan exhausted +[WARN] [1733882054.047050]: Plan exhausted +[WARN] [1733882054.087042]: Plan exhausted +[WARN] [1733882054.127052]: Plan exhausted +[WARN] [1733882054.167037]: Plan exhausted +[WARN] [1733882054.207051]: Plan exhausted +[WARN] [1733882054.247047]: Plan exhausted +[WARN] [1733882054.287087]: Plan exhausted +[WARN] [1733882054.327114]: Plan exhausted +[WARN] [1733882054.367050]: Plan exhausted +[WARN] [1733882054.407066]: Plan exhausted +[WARN] [1733882054.447061]: Plan exhausted +[WARN] [1733882054.487144]: Plan exhausted +[WARN] [1733882054.527057]: Plan exhausted +[WARN] [1733882054.567049]: Plan exhausted +[WARN] [1733882054.607058]: Plan exhausted +[WARN] [1733882054.647049]: Plan exhausted +[WARN] [1733882054.687141]: Plan exhausted +[WARN] [1733882054.727052]: Plan exhausted +[WARN] [1733882054.767101]: Plan exhausted +[WARN] [1733882054.807044]: Plan exhausted +[WARN] [1733882054.847019]: Plan exhausted +[WARN] [1733882054.887033]: Plan exhausted +[WARN] [1733882054.927117]: Plan exhausted +[WARN] [1733882054.967021]: Plan exhausted +[WARN] [1733882055.007079]: Plan exhausted +[WARN] [1733882055.047052]: Plan exhausted +[WARN] [1733882055.087068]: Plan exhausted +[WARN] [1733882055.127060]: Plan exhausted +[WARN] [1733882055.167279]: Plan exhausted +[WARN] [1733882055.207070]: Plan exhausted +[WARN] [1733882055.247038]: Plan exhausted +[WARN] [1733882055.287061]: Plan exhausted +[WARN] [1733882055.327050]: Plan exhausted +[WARN] [1733882055.367044]: Plan exhausted +[WARN] [1733882055.407220]: Plan exhausted +[WARN] [1733882055.447047]: Plan exhausted +[WARN] [1733882055.487638]: Plan exhausted +[WARN] [1733882055.527050]: Plan exhausted +[WARN] [1733882055.567115]: Plan exhausted +[WARN] [1733882055.607057]: Plan exhausted +[WARN] [1733882055.647342]: Plan exhausted +[WARN] [1733882055.687266]: Plan exhausted +[WARN] [1733882055.727048]: Plan exhausted +[WARN] [1733882055.767036]: Plan exhausted +[WARN] [1733882055.807041]: Plan exhausted +[WARN] [1733882055.847427]: Plan exhausted +[WARN] [1733882055.887833]: Plan exhausted +[WARN] [1733882055.927052]: Plan exhausted +[WARN] [1733882055.967049]: Plan exhausted +[WARN] [1733882056.007052]: Plan exhausted +[WARN] [1733882056.047083]: Plan exhausted +[WARN] [1733882056.087589]: Plan exhausted +[WARN] [1733882056.127044]: Plan exhausted +[WARN] [1733882056.167048]: Plan exhausted +[WARN] [1733882056.207262]: Plan exhausted +[WARN] [1733882056.247047]: Plan exhausted +[WARN] [1733882056.287124]: Plan exhausted +[WARN] [1733882056.327050]: Plan exhausted +[WARN] [1733882056.367157]: Plan exhausted +[WARN] [1733882056.407021]: Plan exhausted +[WARN] [1733882056.447050]: Plan exhausted +[WARN] [1733882056.487767]: Plan exhausted +[WARN] [1733882056.527117]: Plan exhausted +[WARN] [1733882056.567130]: Plan exhausted +[WARN] [1733882056.607051]: Plan exhausted +[WARN] [1733882056.647138]: Plan exhausted +[WARN] [1733882056.687124]: Plan exhausted +[WARN] [1733882056.727053]: Plan exhausted +[WARN] [1733882056.767150]: Plan exhausted +[WARN] [1733882056.807051]: Plan exhausted +[WARN] [1733882056.847173]: Plan exhausted +[WARN] [1733882056.887062]: Plan exhausted +[WARN] [1733882056.927035]: Plan exhausted +[WARN] [1733882056.967050]: Plan exhausted +[WARN] [1733882057.007113]: Plan exhausted +[WARN] [1733882057.047042]: Plan exhausted +[WARN] [1733882057.087055]: Plan exhausted +[WARN] [1733882057.127055]: Plan exhausted +[WARN] [1733882057.167049]: Plan exhausted +[WARN] [1733882057.207051]: Plan exhausted +[WARN] [1733882057.247331]: Plan exhausted +[WARN] [1733882057.287073]: Plan exhausted +[WARN] [1733882057.327050]: Plan exhausted +[WARN] [1733882057.367083]: Plan exhausted +[WARN] [1733882057.407049]: Plan exhausted +[WARN] [1733882057.447057]: Plan exhausted +[WARN] [1733882057.487026]: Plan exhausted +[WARN] [1733882057.527054]: Plan exhausted +[WARN] [1733882057.567045]: Plan exhausted +[WARN] [1733882057.607049]: Plan exhausted +[WARN] [1733882057.647047]: Plan exhausted +[WARN] [1733882057.687011]: Plan exhausted +[WARN] [1733882057.727017]: Plan exhausted +[WARN] [1733882057.767211]: Plan exhausted +[WARN] [1733882057.807042]: Plan exhausted +[WARN] [1733882057.847021]: Plan exhausted +[WARN] [1733882057.887207]: Plan exhausted +[WARN] [1733882057.927228]: Plan exhausted +[WARN] [1733882057.967051]: Plan exhausted +[WARN] [1733882058.007057]: Plan exhausted +[WARN] [1733882058.047322]: Plan exhausted +[WARN] [1733882058.087226]: Plan exhausted +[WARN] [1733882058.127048]: Plan exhausted +[WARN] [1733882058.167130]: Plan exhausted +[WARN] [1733882058.207052]: Plan exhausted +[WARN] [1733882058.247214]: Plan exhausted +[WARN] [1733882058.287363]: Plan exhausted +[WARN] [1733882058.327049]: Plan exhausted +[WARN] [1733882058.367048]: Plan exhausted +[WARN] [1733882058.407050]: Plan exhausted +[WARN] [1733882058.447052]: Plan exhausted +[WARN] [1733882058.487058]: Plan exhausted +[WARN] [1733882058.527298]: Plan exhausted +[WARN] [1733882058.567122]: Plan exhausted +[WARN] [1733882058.607055]: Plan exhausted +[WARN] [1733882058.647049]: Plan exhausted +[WARN] [1733882058.687101]: Plan exhausted +[WARN] [1733882058.727047]: Plan exhausted +[WARN] [1733882058.767135]: Plan exhausted +[WARN] [1733882058.807058]: Plan exhausted +[WARN] [1733882058.847335]: Plan exhausted +[WARN] [1733882058.887022]: Plan exhausted +[WARN] [1733882058.927045]: Plan exhausted +[WARN] [1733882058.967049]: Plan exhausted +[WARN] [1733882059.007245]: Plan exhausted +[WARN] [1733882059.047005]: Plan exhausted +[WARN] [1733882059.087120]: Plan exhausted +[WARN] [1733882059.127012]: Plan exhausted +[WARN] [1733882059.167065]: Plan exhausted +[WARN] [1733882059.207019]: Plan exhausted +[WARN] [1733882059.247036]: Plan exhausted +[WARN] [1733882059.287063]: Plan exhausted +[WARN] [1733882059.327055]: Plan exhausted +[WARN] [1733882059.367099]: Plan exhausted +[WARN] [1733882059.407047]: Plan exhausted +[WARN] [1733882059.447119]: Plan exhausted +[WARN] [1733882059.487020]: Plan exhausted +[WARN] [1733882059.527021]: Plan exhausted +[WARN] [1733882059.567276]: Plan exhausted +[WARN] [1733882059.607046]: Plan exhausted +[WARN] [1733882059.647049]: Plan exhausted +[WARN] [1733882059.687052]: Plan exhausted +[WARN] [1733882059.727055]: Plan exhausted +[WARN] [1733882059.767051]: Plan exhausted +[WARN] [1733882059.807057]: Plan exhausted +[WARN] [1733882059.847101]: Plan exhausted +[WARN] [1733882059.887052]: Plan exhausted +[WARN] [1733882059.927545]: Plan exhausted +[WARN] [1733882059.967116]: Plan exhausted +[WARN] [1733882060.007119]: Plan exhausted +[WARN] [1733882060.047050]: Plan exhausted +[WARN] [1733882060.087050]: Plan exhausted +[WARN] [1733882060.127049]: Plan exhausted +[WARN] [1733882060.167005]: Plan exhausted +[WARN] [1733882060.207313]: Plan exhausted +[WARN] [1733882060.247017]: Plan exhausted +[WARN] [1733882060.287113]: Plan exhausted +[WARN] [1733882060.327050]: Plan exhausted +[WARN] [1733882060.367122]: Plan exhausted +[WARN] [1733882060.407054]: Plan exhausted +[WARN] [1733882060.447045]: Plan exhausted +[WARN] [1733882060.487054]: Plan exhausted +[WARN] [1733882060.527052]: Plan exhausted +[WARN] [1733882060.567052]: Plan exhausted +[WARN] [1733882060.607050]: Plan exhausted +[WARN] [1733882060.647051]: Plan exhausted +[WARN] [1733882060.687018]: Plan exhausted +[WARN] [1733882060.727043]: Plan exhausted +[WARN] [1733882060.767119]: Plan exhausted +[WARN] [1733882060.807206]: Plan exhausted +[WARN] [1733882060.847048]: Plan exhausted +[WARN] [1733882060.887264]: Plan exhausted +[WARN] [1733882060.927234]: Plan exhausted +[WARN] [1733882060.967058]: Plan exhausted +[WARN] [1733882061.007091]: Plan exhausted +[WARN] [1733882061.047144]: Plan exhausted +[WARN] [1733882061.087239]: Plan exhausted +[WARN] [1733882061.127246]: Plan exhausted +[WARN] [1733882061.167158]: Plan exhausted +[WARN] [1733882061.207054]: Plan exhausted +[WARN] [1733882061.247165]: Plan exhausted +[WARN] [1733882061.287024]: Plan exhausted +[WARN] [1733882061.327046]: Plan exhausted +[WARN] [1733882061.367183]: Plan exhausted +[WARN] [1733882061.407129]: Plan exhausted +[WARN] [1733882061.447046]: Plan exhausted +[WARN] [1733882061.487052]: Plan exhausted +[WARN] [1733882061.527047]: Plan exhausted +[WARN] [1733882061.567054]: Plan exhausted +[WARN] [1733882061.607161]: Plan exhausted +[WARN] [1733882061.647009]: Plan exhausted +[WARN] [1733882061.687433]: Plan exhausted +[WARN] [1733882061.727046]: Plan exhausted +[WARN] [1733882061.767049]: Plan exhausted +[WARN] [1733882061.807220]: Plan exhausted +[WARN] [1733882061.847046]: Plan exhausted +[WARN] [1733882061.887051]: Plan exhausted +[WARN] [1733882061.927050]: Plan exhausted +[WARN] [1733882061.967053]: Plan exhausted +[WARN] [1733882062.007053]: Plan exhausted +[WARN] [1733882062.047162]: Plan exhausted +[WARN] [1733882062.087059]: Plan exhausted +[WARN] [1733882062.127038]: Plan exhausted +[WARN] [1733882062.167129]: Plan exhausted +[WARN] [1733882062.207045]: Plan exhausted +[WARN] [1733882062.247299]: Plan exhausted +[WARN] [1733882062.287050]: Plan exhausted +[WARN] [1733882062.327052]: Plan exhausted +[WARN] [1733882062.367054]: Plan exhausted +[WARN] [1733882062.407196]: Plan exhausted +[WARN] [1733882062.447055]: Plan exhausted +[WARN] [1733882062.487156]: Plan exhausted +[WARN] [1733882062.527073]: Plan exhausted +[WARN] [1733882062.567137]: Plan exhausted +[WARN] [1733882062.607140]: Plan exhausted +[WARN] [1733882062.647053]: Plan exhausted +[WARN] [1733882062.687056]: Plan exhausted +[WARN] [1733882062.727053]: Plan exhausted +[WARN] [1733882062.767071]: Plan exhausted +[WARN] [1733882062.807082]: Plan exhausted +[WARN] [1733882062.847049]: Plan exhausted +[WARN] [1733882062.887068]: Plan exhausted +[WARN] [1733882062.927056]: Plan exhausted +[WARN] [1733882062.967074]: Plan exhausted +[WARN] [1733882063.007052]: Plan exhausted +[WARN] [1733882063.047042]: Plan exhausted +[WARN] [1733882063.087048]: Plan exhausted +[WARN] [1733882063.127049]: Plan exhausted +[WARN] [1733882063.167047]: Plan exhausted +[WARN] [1733882063.207056]: Plan exhausted +[WARN] [1733882063.247158]: Plan exhausted +[WARN] [1733882063.287093]: Plan exhausted +[WARN] [1733882063.327050]: Plan exhausted +[WARN] [1733882063.367152]: Plan exhausted +[WARN] [1733882063.407056]: Plan exhausted +[WARN] [1733882063.447058]: Plan exhausted +[WARN] [1733882063.487025]: Plan exhausted +[WARN] [1733882063.527024]: Plan exhausted +[WARN] [1733882063.567053]: Plan exhausted +[WARN] [1733882063.607054]: Plan exhausted +[WARN] [1733882063.647300]: Plan exhausted +[WARN] [1733882063.687048]: Plan exhausted +[WARN] [1733882063.727015]: Plan exhausted +[WARN] [1733882063.767130]: Plan exhausted +[WARN] [1733882063.807046]: Plan exhausted +[WARN] [1733882063.847275]: Plan exhausted +[WARN] [1733882063.887052]: Plan exhausted +[WARN] [1733882063.927107]: Plan exhausted +[WARN] [1733882063.967128]: Plan exhausted +[WARN] [1733882064.007052]: Plan exhausted +[WARN] [1733882064.047224]: Plan exhausted +[WARN] [1733882064.087026]: Plan exhausted +[WARN] [1733882064.127050]: Plan exhausted +[WARN] [1733882064.167024]: Plan exhausted +[WARN] [1733882064.207025]: Plan exhausted +[WARN] [1733882064.247044]: Plan exhausted +[WARN] [1733882064.287042]: Plan exhausted +[WARN] [1733882064.327054]: Plan exhausted +[WARN] [1733882064.367053]: Plan exhausted +[WARN] [1733882064.407223]: Plan exhausted +[WARN] [1733882064.447106]: Plan exhausted +[WARN] [1733882064.487052]: Plan exhausted +[WARN] [1733882064.527056]: Plan exhausted +[WARN] [1733882064.567079]: Plan exhausted +[WARN] [1733882064.607060]: Plan exhausted +[WARN] [1733882064.647050]: Plan exhausted +[WARN] [1733882064.687174]: Plan exhausted +[WARN] [1733882064.727053]: Plan exhausted +[WARN] [1733882064.767020]: Plan exhausted +[WARN] [1733882064.807027]: Plan exhausted +[WARN] [1733882064.847050]: Plan exhausted +[WARN] [1733882064.887045]: Plan exhausted +[WARN] [1733882064.927509]: Plan exhausted +[WARN] [1733882064.967138]: Plan exhausted +[WARN] [1733882065.007050]: Plan exhausted +[WARN] [1733882065.047022]: Plan exhausted +[WARN] [1733882065.087052]: Plan exhausted +[WARN] [1733882065.127049]: Plan exhausted +[WARN] [1733882065.167045]: Plan exhausted +[WARN] [1733882065.207147]: Plan exhausted +[WARN] [1733882065.247191]: Plan exhausted +[WARN] [1733882065.287047]: Plan exhausted +[WARN] [1733882065.327043]: Plan exhausted +[WARN] [1733882065.367355]: Plan exhausted +[WARN] [1733882065.407120]: Plan exhausted +[WARN] [1733882065.447181]: Plan exhausted +[WARN] [1733882065.487173]: Plan exhausted +[WARN] [1733882065.527201]: Plan exhausted +[WARN] [1733882065.567048]: Plan exhausted +[WARN] [1733882065.607166]: Plan exhausted +[WARN] [1733882065.647006]: Plan exhausted +[WARN] [1733882065.687023]: Plan exhausted +[WARN] [1733882065.727056]: Plan exhausted +[WARN] [1733882065.767043]: Plan exhausted +[WARN] [1733882065.807048]: Plan exhausted +[WARN] [1733882065.847017]: Plan exhausted +[WARN] [1733882065.887288]: Plan exhausted +[WARN] [1733882065.927184]: Plan exhausted +[WARN] [1733882065.967075]: Plan exhausted +[WARN] [1733882066.007171]: Plan exhausted +[WARN] [1733882066.047332]: Plan exhausted +[WARN] [1733882066.087051]: Plan exhausted +[WARN] [1733882066.127206]: Plan exhausted +[WARN] [1733882066.167326]: Plan exhausted +[WARN] [1733882066.207052]: Plan exhausted +[WARN] [1733882066.247040]: Plan exhausted +[WARN] [1733882066.287051]: Plan exhausted +[WARN] [1733882066.327026]: Plan exhausted +[WARN] [1733882066.367097]: Plan exhausted +[WARN] [1733882066.407017]: Plan exhausted +[WARN] [1733882066.447052]: Plan exhausted +[WARN] [1733882066.487055]: Plan exhausted +[WARN] [1733882066.527056]: Plan exhausted +[WARN] [1733882066.567098]: Plan exhausted +[WARN] [1733882066.607184]: Plan exhausted +[WARN] [1733882066.647047]: Plan exhausted +[WARN] [1733882066.687049]: Plan exhausted +[WARN] [1733882066.727019]: Plan exhausted +[WARN] [1733882066.767239]: Plan exhausted +[WARN] [1733882066.807016]: Plan exhausted +[WARN] [1733882066.847045]: Plan exhausted +[WARN] [1733882066.887052]: Plan exhausted +[WARN] [1733882066.927155]: Plan exhausted +[WARN] [1733882066.967067]: Plan exhausted +[WARN] [1733882067.007022]: Plan exhausted +[WARN] [1733882067.047061]: Plan exhausted +[WARN] [1733882067.087224]: Plan exhausted +[WARN] [1733882067.127199]: Plan exhausted +[WARN] [1733882067.167049]: Plan exhausted +[WARN] [1733882067.207051]: Plan exhausted +[WARN] [1733882067.247069]: Plan exhausted +[WARN] [1733882067.287317]: Plan exhausted +[WARN] [1733882067.327368]: Plan exhausted +[WARN] [1733882067.367091]: Plan exhausted +[WARN] [1733882067.407023]: Plan exhausted +[WARN] [1733882067.447050]: Plan exhausted +[WARN] [1733882067.487056]: Plan exhausted +[WARN] [1733882067.527055]: Plan exhausted +[WARN] [1733882067.578561]: Plan exhausted +[WARN] [1733882067.607018]: Plan exhausted +[WARN] [1733882067.647185]: Plan exhausted +Exception in thread Thread-62: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/FISH/video.py", line 78, in ros_record_cb + self.process.stdin.write(frame.tobytes()) +BrokenPipeError: [Errno 32] Broken pipe +[WARN] [1733882067.687189]: Plan exhausted +[WARN] [1733882067.727179]: Plan exhausted +[WARN] [1733882067.767038]: Plan exhausted +[WARN] [1733882067.807042]: Plan exhausted +[WARN] [1733882067.847068]: Plan exhausted +[WARN] [1733882067.887222]: Plan exhausted +[WARN] [1733882067.927459]: Plan exhausted +[WARN] [1733882067.967060]: Plan exhausted +[WARN] [1733882068.007040]: Plan exhausted +[WARN] [1733882068.047081]: Plan exhausted +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +program exiting gracefully +proceeding to start episode! +[INFO] [1733882069.242753]: Saving video... +[INFO] [1733882069.243578]: Video saved! \ No newline at end of file diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt b/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json b/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c2d5377f7852872b03491b5141b8145b7de12edb --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T01:53:31.224621", + "startedAt": "2024-12-11T01:53:30.842354", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4312.5025, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4500.019, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.007, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.943, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 365.2335090637207 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json b/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..33cb6c6c49d62e5ca82f797f0fdc5270dcffbafb --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/video": {"_type": "video-file", "sha256": "ff4edecb6a17ea039fbf12ba8ab1396b6c7798607c09ee107432df34eedac9aa", "size": 622320, "path": "media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4"}, "global_step": 1, "_timestamp": 1733882069.2541587, "_runtime": 58.40333962440491, "_step": 0, "_wandb": {"runtime": 58}} \ No newline at end of file diff --git a/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log b/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a3b8f02e7c65c420ba1a8d52afd6e7d2ba07e98a --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log @@ -0,0 +1,327 @@ +2024-12-10 20:53:30,850 INFO StreamThr :598006 [internal.py:wandb_internal():85] W&B internal server running at pid: 598006, started at: 2024-12-10 20:53:30.850039 +2024-12-10 20:53:30,851 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:53:30,863 INFO WriterThread:598006 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb +2024-12-10 20:53:30,863 DEBUG SenderThread:598006 [sender.py:send():379] send: header +2024-12-10 20:53:30,875 DEBUG SenderThread:598006 [sender.py:send():379] send: run +2024-12-10 20:53:31,147 INFO SenderThread:598006 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files +2024-12-10 20:53:31,147 INFO SenderThread:598006 [sender.py:_start_run_threads():1188] run started: 52nddla7 with start time 1733882010.850819 +2024-12-10 20:53:31,161 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:53:31,161 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:53:31,206 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:53:31,211 DEBUG HandlerThread:598006 [system_info.py:__init__():26] System info init +2024-12-10 20:53:31,211 DEBUG HandlerThread:598006 [system_info.py:__init__():41] System info init done +2024-12-10 20:53:31,211 INFO HandlerThread:598006 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:53:31,211 INFO SystemMonitor:598006 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:53:31,212 INFO HandlerThread:598006 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:53:31,212 INFO SystemMonitor:598006 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:53:31,212 INFO SystemMonitor:598006 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:53:31,213 INFO SystemMonitor:598006 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:53:31,213 INFO SystemMonitor:598006 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:53:31,214 INFO SystemMonitor:598006 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:53:31,224 DEBUG HandlerThread:598006 [system_info.py:probe():152] Probing system +2024-12-10 20:53:31,226 DEBUG HandlerThread:598006 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:probe():200] Probing system done +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:53:31.224621', 'startedAt': '2024-12-11T01:53:30.842354', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.2335090637207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:53:31,231 INFO HandlerThread:598006 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:53:31,231 INFO HandlerThread:598006 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:53:31,231 DEBUG HandlerThread:598006 [system_info.py:_save_code():44] Saving code +2024-12-10 20:53:31,233 DEBUG HandlerThread:598006 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:53:31,233 DEBUG HandlerThread:598006 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:53:31,305 DEBUG HandlerThread:598006 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:53:31,306 INFO HandlerThread:598006 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:53:31,309 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:53:31,310 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:53:31,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:53:31,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:53:31,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:53:31,571 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/oqlixunj-wandb-metadata.json +2024-12-10 20:53:31,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:31,572 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:53:31,573 INFO HandlerThread:598006 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb" + save: true +} +control { + relay_id: "5d813b8bb5a547c8896a7e03615dca84" +} +_info { + stream_id: "52nddla7" +} + +2024-12-10 20:53:31,574 DEBUG HandlerThread:598006 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:53:31,605 INFO wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/6yv6eek3-code/FISH/eval_robot.py +2024-12-10 20:53:31,648 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/w33l6u66-diff.patch +2024-12-10 20:53:31,702 DEBUG SenderThread:598006 [sender.py:send():379] send: telemetry +2024-12-10 20:53:31,702 DEBUG SenderThread:598006 [sender.py:send():379] send: tbrecord +2024-12-10 20:53:31,703 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:53:31,703 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 with policy live +2024-12-10 20:53:31,907 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/uat8jiac-events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH +2024-12-10 20:53:32,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code +2024-12-10 20:53:32,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:33,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:34,148 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:34,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:35,547 DEBUG SenderThread:598006 [sender.py:send():379] send: config +2024-12-10 20:53:35,590 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:36,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:36,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:37,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:38,149 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:38,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:39,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:40,150 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:40,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:41,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:42,082 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:42,150 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:42,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:43,572 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:44,151 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:44,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:45,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:46,151 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:53:46,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:53:46,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:53:46,614 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:47,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:47,640 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:48,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:49,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:50,573 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:51,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:52,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:53,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:53,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:54,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:55,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:56,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:57,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:58,571 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:53:58,574 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:53:59,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:00,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:01,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:01,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:01,613 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:02,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:03,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:03,722 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:04,156 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml +2024-12-10 20:54:04,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:05,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:06,575 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:07,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:08,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:09,550 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:09,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:10,158 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:10,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:11,576 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:12,159 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:12,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:13,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:14,159 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:14,567 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:14,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:15,160 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:15,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:16,570 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:16,570 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:16,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:17,160 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:17,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:18,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:19,161 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:19,567 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:19,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:20,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:21,161 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:21,577 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:22,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:23,162 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:23,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:24,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:24,607 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:25,162 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:25,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:26,578 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:27,163 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:27,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:28,579 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:29,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:29,254 DEBUG SenderThread:598006 [sender.py:send():379] send: files +2024-12-10 20:54:29,254 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 with policy now +2024-12-10 20:54:29,256 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:54:29,257 DEBUG SenderThread:598006 [sender.py:send():379] send: history +2024-12-10 20:54:29,257 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:54:29,257 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:29,258 DEBUG SenderThread:598006 [sender.py:send():379] send: exit +2024-12-10 20:54:29,258 INFO SenderThread:598006 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_exit():588] handling runtime: 58 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_exit():594] send defer +2024-12-10 20:54:29,259 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,259 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:54:29,259 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:54:29,259 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:54:29,259 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,259 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:54:29,260 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,260 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:54:29,260 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:54:29,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,260 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:54:29,260 INFO HandlerThread:598006 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:54:29,260 DEBUG SystemMonitor:598006 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:54:29,269 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:54:29,269 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:54:29,319 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:54:29,319 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:54:29,320 INFO HandlerThread:598006 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:54:29,320 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,320 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:54:29,320 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:54:29,320 DEBUG SenderThread:598006 [sender.py:send():379] send: stats +2024-12-10 20:54:29,320 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,320 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:54:29,321 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:29,321 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:54:29,321 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:54:29,321 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:29,321 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:54:29,518 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/0kw42edo-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:30,164 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval +2024-12-10 20:54:30,165 INFO Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media +2024-12-10 20:54:30,523 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/ks8s91l3-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:54:35,581 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,581 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,581 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:54:35,581 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:54:35,581 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:35,582 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,582 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:54:35,582 DEBUG SenderThread:598006 [sender.py:send():379] send: summary +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:54:35,582 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:54:35,582 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,583 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:54:35,583 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:54:35,583 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:35,583 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:54:35,583 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:35,583 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:54:36,166 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:36,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:37,110 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:54:37,110 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:37,110 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:37,110 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:54:37,111 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:37,111 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:54:37,111 INFO SenderThread:598006 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:54:37,113 INFO SenderThread:598006 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:54:37,127 INFO SenderThread:598006 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:54:37,136 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:54:37,136 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:37,136 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:54:37,137 DEBUG SenderThread:598006 [sender.py:send():379] send: artifact +2024-12-10 20:54:37,166 INFO Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:37,260 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:37,780 INFO wandb-upload_1:598006 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_p_p260o +2024-12-10 20:54:37,783 INFO wandb-upload_0:598006 [upload_job.py:push():85] Skipped uploading /tmp/tmppldao70p/wandb-job.json +2024-12-10 20:54:37,954 INFO wandb-upload_2:598006 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4q7vuxho +2024-12-10 20:54:38,567 INFO SenderThread:598006 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'versionIndex': 336}}} +2024-12-10 20:54:38,567 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:38,567 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:54:38,567 INFO SenderThread:598006 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:54:39,167 INFO SenderThread:598006 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml config.yaml +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json wandb-summary.json +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt requirements.txt +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:54:39,168 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log output.log +2024-12-10 20:54:39,173 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch diff.patch +2024-12-10 20:54:39,173 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:54:39,174 INFO SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 +2024-12-10 20:54:39,175 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:54:39,175 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,175 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,175 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:54:39,175 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,175 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:54:39,175 INFO SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:54:39,261 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,261 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,363 INFO wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json +2024-12-10 20:54:39,397 INFO wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt +2024-12-10 20:54:39,416 INFO wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml +2024-12-10 20:54:39,447 INFO wandb-upload_3:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log +2024-12-10 20:54:39,647 INFO Thread-11 (_thread_body):598006 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:54:39,647 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,647 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:54:39,648 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,648 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:54:39,648 INFO SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:54:39,648 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:54:39,648 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,648 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:54:39,648 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,649 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:54:39,649 INFO SenderThread:598006 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:54:39,709 INFO SenderThread:598006 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:54:39,709 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,709 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:54:39,709 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:54:39,709 INFO SenderThread:598006 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:54:39,709 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:54:39,709 INFO HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send():379] send: final +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send():379] send: footer +2024-12-10 20:54:39,710 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: defer +2024-12-10 20:54:39,710 INFO SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:54:39,710 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:54:39,711 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,711 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:54:39,712 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:54:39,713 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:54:39,713 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:39,793 DEBUG SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:54:39,794 INFO MainThread:598006 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:54:39,794 DEBUG HandlerThread:598006 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:54:39,794 INFO HandlerThread:598006 [handler.py:finish():882] shutting down handler +2024-12-10 20:54:40,712 INFO WriterThread:598006 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb +2024-12-10 20:54:40,794 INFO SenderThread:598006 [sender.py:finish():1615] shutting down sender +2024-12-10 20:54:40,794 INFO SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:54:40,794 INFO SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher diff --git a/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log b/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4ab29e48de854cde31d3ccafd386e4305f1235a5 --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Configure stats pid to 597884 +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/settings +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():619] starting backend +2024-12-10 20:53:30,845 INFO MainThread:597884 [wandb_init.py:init():623] setting up manager +2024-12-10 20:53:30,849 INFO MainThread:597884 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:53:30,850 INFO MainThread:597884 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:53:30,862 INFO MainThread:597884 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:53:30,871 INFO MainThread:597884 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:53:31,160 INFO MainThread:597884 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:53:31,200 INFO MainThread:597884 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:53:31,201 INFO MainThread:597884 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:53:31,571 INFO MainThread:597884 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:53:31,573 INFO MainThread:597884 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:53:31,573 INFO MainThread:597884 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb, True +2024-12-10 20:53:35,546 INFO MainThread:597884 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:54:40,795 WARNING MsgRouterThr:597884 [router.py:message_loop():77] message_loop has been closed diff --git a/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb b/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ea920df4d40a658206c6127e59e2242e4d323b87 Binary files /dev/null and b/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb differ diff --git a/205451/.hydra/config.yaml b/205451/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/205451/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205451/.hydra/hydra.yaml b/205451/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dca13362a2e6a054cbb65715e6666bc4ea2e9e28 --- /dev/null +++ b/205451/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205451/.hydra/overrides.yaml b/205451/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205451/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205451/episode_rosbags/aligned_depth_to_color_K.npy b/205451/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205451/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205451/episode_rosbags/cam_tf_world.npy b/205451/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/205451/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/205451/episode_rosbags/color_K.npy b/205451/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205451/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205451/episode_rosbags/depth_K.npy b/205451/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/205451/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag b/205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag new file mode 100644 index 0000000000000000000000000000000000000000..0c34251e30ff75b2f46e9d5ab9d142aea1bc2b65 --- /dev/null +++ b/205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d73eecf21e920185f978b95d07f1da9985af31f10f5817abf59711dc9595111c +size 115247987 diff --git a/205451/eval_robot.log b/205451/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ca0a125a00237d41be63d36d0a0163056936fc90 --- /dev/null +++ b/205451/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 20:54:52,007][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:54:52,009][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/205451/eval_video/0_eval.mp4 b/205451/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ace72ddcbe13cbc515bacb1937b5daf81c20736c --- /dev/null +++ b/205451/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb699cd8c28a00c2c3484406ce2007e10bf39f065783c0f23eb8419b1825650 +size 120808 diff --git a/205451/tb/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 b/205451/tb/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 new file mode 100644 index 0000000000000000000000000000000000000000..b2c612d73029bce1a19d72e682660a709bf39a1b --- /dev/null +++ b/205451/tb/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a5eafd7b32576e72707afb0c92f7dbf3cfdd45f5c3d38b4b350d858fbca964 +size 88 diff --git a/205451/wandb/debug-internal.log b/205451/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b246d302f2df13b93d49ffcfdab5e45bf002cb51 --- /dev/null +++ b/205451/wandb/debug-internal.log @@ -0,0 +1,429 @@ +2024-12-10 20:54:57,845 INFO StreamThr :598427 [internal.py:wandb_internal():85] W&B internal server running at pid: 598427, started at: 2024-12-10 20:54:57.844685 +2024-12-10 20:54:57,846 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:54:57,856 INFO WriterThread:598427 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb +2024-12-10 20:54:57,856 DEBUG SenderThread:598427 [sender.py:send():379] send: header +2024-12-10 20:54:57,868 DEBUG SenderThread:598427 [sender.py:send():379] send: run +2024-12-10 20:54:58,148 INFO SenderThread:598427 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files +2024-12-10 20:54:58,148 INFO SenderThread:598427 [sender.py:_start_run_threads():1188] run started: 52qyo5rz with start time 1733882097.844997 +2024-12-10 20:54:58,164 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:54:58,164 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:54:58,239 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:54:58,243 DEBUG HandlerThread:598427 [system_info.py:__init__():26] System info init +2024-12-10 20:54:58,243 DEBUG HandlerThread:598427 [system_info.py:__init__():41] System info init done +2024-12-10 20:54:58,243 INFO HandlerThread:598427 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:54:58,244 INFO SystemMonitor:598427 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:54:58,244 INFO HandlerThread:598427 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:54:58,244 INFO SystemMonitor:598427 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:54:58,245 INFO SystemMonitor:598427 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:54:58,245 INFO SystemMonitor:598427 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:54:58,246 INFO SystemMonitor:598427 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:54:58,247 INFO SystemMonitor:598427 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:54:58,257 DEBUG HandlerThread:598427 [system_info.py:probe():152] Probing system +2024-12-10 20:54:58,260 DEBUG HandlerThread:598427 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:probe():200] Probing system done +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:54:58.257274', 'startedAt': '2024-12-11T01:54:57.836911', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4499.9932499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.827, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.9410629272461}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:54:58,267 INFO HandlerThread:598427 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:54:58,267 INFO HandlerThread:598427 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:_save_code():44] Saving code +2024-12-10 20:54:58,268 DEBUG HandlerThread:598427 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:54:58,268 DEBUG HandlerThread:598427 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:54:58,331 DEBUG HandlerThread:598427 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:54:58,331 INFO HandlerThread:598427 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:54:58,334 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:54:58,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:54:58,562 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:54:58,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:58,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:58,565 INFO HandlerThread:598427 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/tb" + save: true +} +control { + relay_id: "d64c4285f6dd41fbb05315d418d07145" +} +_info { + stream_id: "52qyo5rz" +} + +2024-12-10 20:54:58,566 DEBUG HandlerThread:598427 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:54:58,618 INFO wandb-upload_0:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/h61fh9rs-wandb-metadata.json +2024-12-10 20:54:58,619 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/7onuwb17-diff.patch +2024-12-10 20:54:58,676 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/a7ljciuc-code/FISH/eval_robot.py +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: telemetry +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: tbrecord +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:54:58,717 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 with policy live +2024-12-10 20:54:58,917 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/qgclrf9j-events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:54:59,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:54:59,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH +2024-12-10 20:54:59,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:00,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:01,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:01,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:02,122 DEBUG SenderThread:598427 [sender.py:send():379] send: config +2024-12-10 20:55:02,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:03,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:03,166 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:03,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:04,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:05,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:05,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:06,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:07,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:08,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:08,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:09,151 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:09,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:10,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:11,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:12,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:13,152 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:13,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:13,562 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:13,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:13,692 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:14,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:15,153 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:15,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:16,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:17,153 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:17,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:18,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:19,154 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:19,431 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:19,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:20,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:21,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:21,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:22,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:22,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:23,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:23,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:24,156 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:24,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:24,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:25,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:26,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:27,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:28,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:28,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:28,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:29,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:29,660 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:30,157 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml +2024-12-10 20:55:30,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:31,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:32,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:33,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:34,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:35,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:35,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:36,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:37,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:38,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:39,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:40,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:40,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:41,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:42,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:43,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:43,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:43,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:44,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:45,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:45,706 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:46,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:47,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:48,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:49,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:50,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:51,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:51,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:52,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:53,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:54,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:55,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:56,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:57,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:57,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:58,247 DEBUG SystemMonitor:598427 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:55:58,247 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:55:58,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:58,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:59,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:00,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:01,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:02,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:02,713 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:03,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:04,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:05,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:06,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:07,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:08,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:08,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:09,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:10,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:11,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:12,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:13,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:13,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:13,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:13,719 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:14,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:15,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:16,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:17,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:18,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:19,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:19,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:20,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:21,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:22,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:23,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:24,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:24,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:25,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:26,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:27,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:28,248 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:56:28,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:28,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:28,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:29,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:29,721 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:30,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:31,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:32,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:33,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:34,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:35,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:35,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:36,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:37,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:38,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:39,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:40,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:40,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:41,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:42,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:43,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:43,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:43,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:44,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:45,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:45,639 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:46,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:47,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:48,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:49,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:50,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:51,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:51,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:52,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:53,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:54,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:55,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:56,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:56,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:57,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:58,249 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:56:58,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:58,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:59,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:00,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:01,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:01,643 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:02,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:03,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:04,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:05,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:06,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:07,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:07,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:08,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:09,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:10,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:11,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:12,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:12,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:13,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:57:13,564 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:57:13,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:14,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:15,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:16,580 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:17,580 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:17,655 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:17,970 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:57:17,971 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 with policy now +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send():379] send: history +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:57:17,972 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:17,973 DEBUG SenderThread:598427 [sender.py:send():379] send: exit +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:send_exit():588] handling runtime: 139 +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:17,974 INFO SenderThread:598427 [sender.py:send_exit():594] send defer +2024-12-10 20:57:17,974 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,974 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:57:17,974 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:57:17,978 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,978 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:57:17,978 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:57:17,979 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:57:17,979 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,979 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:57:17,979 INFO HandlerThread:598427 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:57:17,979 DEBUG SystemMonitor:598427 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:57:17,979 DEBUG SystemMonitor:598427 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:57:17,980 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:57:17,980 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:57:18,047 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:18,047 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:57:18,047 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:57:18,047 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:57:18,047 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:18,047 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:57:18,048 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:18,048 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:57:18,048 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:57:18,048 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:18,048 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos +2024-12-10 20:57:18,231 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/483yx6ns-media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:18,499 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/0erhu0sx-media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:19,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:57:24,581 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,581 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:57:24,581 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:57:24,581 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:24,582 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,582 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:57:24,582 DEBUG SenderThread:598427 [sender.py:send():379] send: summary +2024-12-10 20:57:24,582 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:24,582 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,582 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,583 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:57:24,583 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,583 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:57:24,583 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:57:24,971 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:25,189 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:25,563 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:57:25,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:25,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:25,563 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:57:25,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:25,563 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:57:25,563 INFO SenderThread:598427 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:57:25,566 INFO SenderThread:598427 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:57:25,580 INFO SenderThread:598427 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:57:25,594 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:57:25,595 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:25,595 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:57:25,595 DEBUG SenderThread:598427 [sender.py:send():379] send: artifact +2024-12-10 20:57:25,920 INFO SenderThread:598427 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'versionIndex': 337}}} +2024-12-10 20:57:25,920 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:25,920 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:57:25,920 INFO SenderThread:598427 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:57:25,972 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,190 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml config.yaml +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json wandb-summary.json +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt requirements.txt +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:57:26,191 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:57:26,192 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log output.log +2024-12-10 20:57:26,192 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch diff.patch +2024-12-10 20:57:26,197 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:57:26,197 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:26,197 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:57:26,197 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,197 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,197 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:57:26,198 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,198 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:57:26,198 INFO SenderThread:598427 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:57:26,381 INFO wandb-upload_3:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt +2024-12-10 20:57:26,402 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml +2024-12-10 20:57:26,418 INFO wandb-upload_0:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:26,447 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:26,647 INFO Thread-11 (_thread_body):598427 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:57:26,647 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,647 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:57:26,648 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:57:26,648 INFO SenderThread:598427 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:57:26,648 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,648 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:57:26,648 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:57:26,648 INFO SenderThread:598427 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:57:26,855 INFO SenderThread:598427 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:57:26,855 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:57:26,855 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,855 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:57:26,856 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,856 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send():379] send: final +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send():379] send: footer +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,857 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:57:26,857 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:26,913 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:57:26,913 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:57:26,914 INFO HandlerThread:598427 [handler.py:finish():882] shutting down handler +2024-12-10 20:57:27,859 INFO WriterThread:598427 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb +2024-12-10 20:57:27,913 INFO SenderThread:598427 [sender.py:finish():1615] shutting down sender +2024-12-10 20:57:27,913 INFO SenderThread:598427 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:57:27,913 INFO SenderThread:598427 [file_pusher.py:join():175] waiting for file pusher diff --git a/205451/wandb/debug.log b/205451/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..39fe8b9c15afd5f65751757771ee29f6c7b8960f --- /dev/null +++ b/205451/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Configure stats pid to 598305 +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/settings +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug.log +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug-internal.log +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():619] starting backend +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():623] setting up manager +2024-12-10 20:54:57,843 INFO MainThread:598305 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:54:57,844 INFO MainThread:598305 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:54:57,856 INFO MainThread:598305 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:54:57,866 INFO MainThread:598305 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:54:58,163 INFO MainThread:598305 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:54:58,232 INFO MainThread:598305 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:54:58,233 INFO MainThread:598305 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:54:58,565 INFO MainThread:598305 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:54:58,565 INFO MainThread:598305 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/tb, True +2024-12-10 20:55:02,121 INFO MainThread:598305 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:57:27,914 WARNING MsgRouterThr:598305 [router.py:message_loop():77] message_loop has been closed diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py b/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml b/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59fa020c11f14a298f6654f569e825e928168b02 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733882097 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch b/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..ffeda89f47c1d6b36e048eac4bf0463d100c7103 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch @@ -0,0 +1,34 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..aa52de5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos +@@ -151,7 +156,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history + # wandb_run_id: '250380_0' # RGBD+mask+contact+act history + # wandb_run_id: '250370_0' # RGBD+contact+act history + +-true_action_history: true ++true_action_history: false + wandb_notes: null + + checkpoint_epoch: 12000 # for depth + all contact diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 b/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ace72ddcbe13cbc515bacb1937b5daf81c20736c --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb699cd8c28a00c2c3484406ce2007e10bf39f065783c0f23eb8419b1825650 +size 120808 diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log b/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b3faaefd69d065d52de1f96f939df355cba35bbe --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log @@ -0,0 +1,128 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733882102.122582]: resetting environment +[INFO] [1733882102.130173]: cleared current plan +[INFO] [1733882102.131216]: moving to home +[INFO] [1733882104.033202]: reached home +[INFO] [1733882104.034532]: reset action history +[INFO] [1733882106.742985]: environment reset +Starting episode 0 +[INFO] [1733882106.745780]: resetting environment +[INFO] [1733882106.753376]: cleared current plan +[INFO] [1733882106.754234]: moving to home +[INFO] [1733882107.756475]: reached home +[INFO] [1733882107.758160]: reset action history +[INFO] [1733882110.466464]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733882113.865792]: resetting environment +[INFO] [1733882113.869902]: cleared current plan +[INFO] [1733882113.870681]: moving to home +[INFO] [1733882114.872976]: reached home +[INFO] [1733882114.874325]: reset action history +[INFO] [1733882117.581810]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-598305.log +started roslaunch server http://158.130.50.37:33047/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [598528] +process[print_text-2]: started with pid [598551] +process[rosbag_record-3]: started with pid [598553] +started bagging! +[INFO] [1733882118.350041]: received mask_obs with header seq: 112 +stamp: + secs: 1733882118 + nsecs: 349298954 +frame_id: '' and grasped_object_mask header seq: 13379 +stamp: + secs: 1733882118 + nsecs: 88709831 +frame_id: "camera_color_optical_frame" +[INFO] [1733882118.523886]: received mask_obs with header seq: 113 +stamp: + secs: 1733882118 + nsecs: 523519992 +frame_id: '' and grasped_object_mask header seq: 13386 +stamp: + secs: 1733882118 + nsecs: 324564934 +frame_id: "camera_color_optical_frame" +Exception in thread Thread-71: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 562, in ros_step_cb + action_history, plan_headers_list = self.arm.get_action_history( + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 574, in get_action_history + action_history, plan_headers_list = self.get_hallucinated_action_history(prev_plan_timestamp, current_timestamp, history_length) + File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 599, in get_hallucinated_action_history + assert action_plan_return is not None, f'No plan found after {timeout.to_sec()} seconds' +AssertionError: No plan found after 1.0 seconds +[WARN] [1733882119.507797]: Plan exhausted +[WARN] [1733882119.547786]: Plan exhausted +[WARN] [1733882119.587786]: Plan exhausted +[WARN] [1733882119.627784]: Plan exhausted +[WARN] [1733882119.667951]: Plan exhausted +[WARN] [1733882119.707792]: Plan exhausted +[WARN] [1733882119.747981]: Plan exhausted +[WARN] [1733882119.787757]: Plan exhausted +[WARN] [1733882119.827780]: Plan exhausted +[WARN] [1733882119.867917]: Plan exhausted +[WARN] [1733882119.907781]: Plan exhausted +[WARN] [1733882119.947895]: Plan exhausted +[WARN] [1733882119.987787]: Plan exhausted +[WARN] [1733882120.027787]: Plan exhausted +[WARN] [1733882120.067819]: Plan exhausted +[WARN] [1733882120.107747]: Plan exhausted +[WARN] [1733882120.147855]: Plan exhausted +[WARN] [1733882120.187791]: Plan exhausted +[WARN] [1733882120.227782]: Plan exhausted +[WARN] [1733882120.268870]: Plan exhausted +[WARN] [1733882120.307781]: Plan exhausted +[WARN] [1733882120.347733]: Plan exhausted +[WARN] [1733882120.387793]: Plan exhausted +[WARN] [1733882120.427838]: Plan exhausted +[WARN] [1733882120.467769]: Plan exhausted +[WARN] [1733882120.507955]: Plan exhausted +[WARN] [1733882120.547909]: Plan exhausted +[WARN] [1733882120.587762]: Plan exhausted +Exception in thread Thread-62: +Traceback (most recent call last): + File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run + self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration)) + File "/home/leonmkim/fish_leon/FISH/video.py", line 78, in ros_record_cb + self.process.stdin.write(frame.tobytes()) +BrokenPipeError: [Errno 32] Broken pipe +[WARN] [1733882120.627783]: Plan exhausted +[WARN] [1733882120.668061]: Plan exhausted +[WARN] [1733882120.708108]: Plan exhausted +[WARN] [1733882120.747794]: Plan exhausted +[WARN] [1733882120.787749]: Plan exhausted +[WARN] [1733882120.827763]: Plan exhausted +[WARN] [1733882120.868249]: Plan exhausted +[WARN] [1733882120.907827]: Plan exhausted +[rosbag_record-3] killing on exit[pub_text-1] killing on exit[print_text-2] killing on exit +proceeding to start episode! +program exiting gracefully +[WARN] [1733882120.947781]: Plan exhausted +[WARN] [1733882120.987790]: Plan exhausted +[WARN] [1733882121.027782]: Plan exhausted +[WARN] [1733882121.068702]: Plan exhausted +[WARN] [1733882121.107785]: Plan exhausted +[WARN] [1733882121.147948]: Plan exhausted +proceeding to start episode! +[INFO] [1733882237.928490]: Saving video... +[INFO] [1733882237.928839]: Video saved! \ No newline at end of file diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt b/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json b/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ab365ef64f0860b7ec012bf9c2e4e400a3b05651 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T01:54:58.257274", + "startedAt": "2024-12-11T01:54:57.836911", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4499.9932499999995, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.827, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.002, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 365.9410629272461 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json b/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..657d5cb115eee897f159d52a12cffbcbd5f8fe73 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/video": {"_type": "video-file", "sha256": "bfb699cd8c28a00c2c3484406ce2007e10bf39f065783c0f23eb8419b1825650", "size": 120808, "path": "media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4"}, "global_step": 1, "_timestamp": 1733882237.9356365, "_runtime": 140.09063959121704, "_step": 0, "_wandb": {"runtime": 139}} \ No newline at end of file diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug-internal.log b/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b246d302f2df13b93d49ffcfdab5e45bf002cb51 --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug-internal.log @@ -0,0 +1,429 @@ +2024-12-10 20:54:57,845 INFO StreamThr :598427 [internal.py:wandb_internal():85] W&B internal server running at pid: 598427, started at: 2024-12-10 20:54:57.844685 +2024-12-10 20:54:57,846 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status +2024-12-10 20:54:57,856 INFO WriterThread:598427 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb +2024-12-10 20:54:57,856 DEBUG SenderThread:598427 [sender.py:send():379] send: header +2024-12-10 20:54:57,868 DEBUG SenderThread:598427 [sender.py:send():379] send: run +2024-12-10 20:54:58,148 INFO SenderThread:598427 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files +2024-12-10 20:54:58,148 INFO SenderThread:598427 [sender.py:_start_run_threads():1188] run started: 52qyo5rz with start time 1733882097.844997 +2024-12-10 20:54:58,164 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 20:54:58,164 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: check_version +2024-12-10 20:54:58,239 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 20:54:58,243 DEBUG HandlerThread:598427 [system_info.py:__init__():26] System info init +2024-12-10 20:54:58,243 DEBUG HandlerThread:598427 [system_info.py:__init__():41] System info init done +2024-12-10 20:54:58,243 INFO HandlerThread:598427 [system_monitor.py:start():194] Starting system monitor +2024-12-10 20:54:58,244 INFO SystemMonitor:598427 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 20:54:58,244 INFO HandlerThread:598427 [system_monitor.py:probe():214] Collecting system info +2024-12-10 20:54:58,244 INFO SystemMonitor:598427 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 20:54:58,245 INFO SystemMonitor:598427 [interfaces.py:start():188] Started disk monitoring +2024-12-10 20:54:58,245 INFO SystemMonitor:598427 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 20:54:58,246 INFO SystemMonitor:598427 [interfaces.py:start():188] Started memory monitoring +2024-12-10 20:54:58,247 INFO SystemMonitor:598427 [interfaces.py:start():188] Started network monitoring +2024-12-10 20:54:58,257 DEBUG HandlerThread:598427 [system_info.py:probe():152] Probing system +2024-12-10 20:54:58,260 DEBUG HandlerThread:598427 [system_info.py:_probe_git():137] Probing git +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:_probe_git():145] Probing git done +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:probe():200] Probing system done +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:54:58.257274', 'startedAt': '2024-12-11T01:54:57.836911', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4499.9932499999995, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.827, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.9410629272461}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 20:54:58,267 INFO HandlerThread:598427 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 20:54:58,267 INFO HandlerThread:598427 [system_monitor.py:probe():227] Publishing system info +2024-12-10 20:54:58,267 DEBUG HandlerThread:598427 [system_info.py:_save_code():44] Saving code +2024-12-10 20:54:58,268 DEBUG HandlerThread:598427 [system_info.py:_save_code():67] Saving code done +2024-12-10 20:54:58,268 DEBUG HandlerThread:598427 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 20:54:58,331 DEBUG HandlerThread:598427 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 20:54:58,331 INFO HandlerThread:598427 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 20:54:58,334 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 20:54:58,334 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 20:54:58,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 20:54:58,562 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: python_packages +2024-12-10 20:54:58,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:54:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:54:58,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:54:58,565 INFO HandlerThread:598427 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/tb" + save: true +} +control { + relay_id: "d64c4285f6dd41fbb05315d418d07145" +} +_info { + stream_id: "52qyo5rz" +} + +2024-12-10 20:54:58,566 DEBUG HandlerThread:598427 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 20:54:58,618 INFO wandb-upload_0:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/h61fh9rs-wandb-metadata.json +2024-12-10 20:54:58,619 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/7onuwb17-diff.patch +2024-12-10 20:54:58,676 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/a7ljciuc-code/FISH/eval_robot.py +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: telemetry +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: tbrecord +2024-12-10 20:54:58,716 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:54:58,717 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 with policy live +2024-12-10 20:54:58,917 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/qgclrf9j-events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:54:59,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:54:59,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code +2024-12-10 20:54:59,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH +2024-12-10 20:54:59,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:00,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:01,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:01,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:02,122 DEBUG SenderThread:598427 [sender.py:send():379] send: config +2024-12-10 20:55:02,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:03,149 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:03,166 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:03,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:04,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:05,150 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:05,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:06,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:07,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:08,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:08,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:09,151 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:09,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:10,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:11,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:12,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:13,152 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:13,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:13,562 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:13,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:13,692 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:14,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:15,153 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:15,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:16,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:17,153 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:17,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:18,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:19,154 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:19,431 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:19,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:20,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:21,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:21,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:22,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:22,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:23,155 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:23,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:24,156 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:55:24,564 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:24,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:25,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:26,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:27,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:28,562 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:28,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:28,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:29,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:29,660 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:30,157 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml +2024-12-10 20:55:30,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:31,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:32,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:33,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:34,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:35,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:35,568 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:36,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:37,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:38,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:39,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:40,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:40,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:41,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:42,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:43,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:43,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:43,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:44,569 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:45,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:45,706 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:46,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:47,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:48,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:49,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:50,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:51,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:51,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:52,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:53,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:54,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:55,570 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:56,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:57,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:55:57,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:58,247 DEBUG SystemMonitor:598427 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 20:55:58,247 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:55:58,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:55:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:55:58,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:55:59,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:00,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:01,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:02,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:02,713 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:03,571 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:04,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:05,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:06,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:07,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:08,565 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:08,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:09,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:10,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:11,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:12,572 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:13,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:13,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:13,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:13,719 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:14,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:15,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:16,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:17,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:18,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:19,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:19,573 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:20,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:21,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:22,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:23,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:24,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:24,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:25,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:26,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:27,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:28,248 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:56:28,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:28,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:28,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:29,574 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:29,721 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:30,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:31,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:32,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:33,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:34,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:35,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:35,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:36,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:37,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:38,575 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:39,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:40,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:40,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:41,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:42,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:43,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:43,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:43,606 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:44,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:45,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:45,639 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:46,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:47,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:48,576 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:49,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:50,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:51,566 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:51,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:52,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:53,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:54,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:55,577 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:56,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:56:56,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:57,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:58,249 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:56:58,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:56:58,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:56:58,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:56:59,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:00,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:01,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:01,643 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:02,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:03,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:04,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:05,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:06,578 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:07,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:07,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:08,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:09,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:10,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:11,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:12,567 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:12,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:13,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 20:57:13,564 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: stop_status +2024-12-10 20:57:13,605 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:14,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:15,579 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:16,580 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:17,580 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:17,655 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:17,970 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send():379] send: files +2024-12-10 20:57:17,971 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 with policy now +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send():379] send: history +2024-12-10 20:57:17,971 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: summary_record +2024-12-10 20:57:17,972 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:17,973 DEBUG SenderThread:598427 [sender.py:send():379] send: exit +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:send_exit():588] handling runtime: 139 +2024-12-10 20:57:17,973 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:17,974 INFO SenderThread:598427 [sender.py:send_exit():594] send defer +2024-12-10 20:57:17,974 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,974 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 20:57:17,974 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 1 +2024-12-10 20:57:17,978 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,978 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 20:57:17,978 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:17,978 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 20:57:17,979 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 2 +2024-12-10 20:57:17,979 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:17,979 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 20:57:17,979 INFO HandlerThread:598427 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 20:57:17,979 DEBUG SystemMonitor:598427 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 20:57:17,979 DEBUG SystemMonitor:598427 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 20:57:17,980 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 20:57:17,980 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 20:57:18,046 INFO HandlerThread:598427 [interfaces.py:finish():200] Joined network monitor +2024-12-10 20:57:18,047 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:18,047 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 20:57:18,047 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 3 +2024-12-10 20:57:18,047 DEBUG SenderThread:598427 [sender.py:send():379] send: stats +2024-12-10 20:57:18,047 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:18,047 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 20:57:18,048 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:18,048 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 20:57:18,048 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 4 +2024-12-10 20:57:18,048 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:18,048 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval +2024-12-10 20:57:18,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos +2024-12-10 20:57:18,231 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/483yx6ns-media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:18,499 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /tmp/tmphc14acp_wandb/0erhu0sx-media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:19,187 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:24,581 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 20:57:24,581 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,581 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 20:57:24,581 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 5 +2024-12-10 20:57:24,581 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:24,582 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,582 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 20:57:24,582 DEBUG SenderThread:598427 [sender.py:send():379] send: summary +2024-12-10 20:57:24,582 INFO SenderThread:598427 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 20:57:24,582 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,582 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 6 +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,583 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 20:57:24,583 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 7 +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 20:57:24,583 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:24,583 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 20:57:24,583 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:24,583 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 20:57:24,971 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:25,189 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:25,563 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 8 +2024-12-10 20:57:25,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:25,563 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:25,563 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 20:57:25,563 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:25,563 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 20:57:25,563 INFO SenderThread:598427 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 20:57:25,566 INFO SenderThread:598427 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 20:57:25,580 INFO SenderThread:598427 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 20:57:25,594 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 9 +2024-12-10 20:57:25,595 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:25,595 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 20:57:25,595 DEBUG SenderThread:598427 [sender.py:send():379] send: artifact +2024-12-10 20:57:25,920 INFO SenderThread:598427 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'versionIndex': 337}}} +2024-12-10 20:57:25,920 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:25,920 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 20:57:25,920 INFO SenderThread:598427 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 20:57:25,972 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,190 INFO Thread-12 :598427 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml config.yaml +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json wandb-summary.json +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt requirements.txt +2024-12-10 20:57:26,190 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-metadata.json wandb-metadata.json +2024-12-10 20:57:26,191 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 events.out.tfevents.1733882098.leonmkim-ROG-Strix-G15CS-G15CS.598305.0 +2024-12-10 20:57:26,192 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log output.log +2024-12-10 20:57:26,192 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/diff.patch diff.patch +2024-12-10 20:57:26,197 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 20:57:26,197 INFO SenderThread:598427 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 +2024-12-10 20:57:26,197 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 10 +2024-12-10 20:57:26,197 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,197 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,197 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 20:57:26,198 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,198 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 20:57:26,198 INFO SenderThread:598427 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:57:26,381 INFO wandb-upload_3:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/requirements.txt +2024-12-10 20:57:26,402 INFO wandb-upload_1:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/config.yaml +2024-12-10 20:57:26,418 INFO wandb-upload_0:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/wandb-summary.json +2024-12-10 20:57:26,447 INFO wandb-upload_2:598427 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/files/output.log +2024-12-10 20:57:26,647 INFO Thread-11 (_thread_body):598427 [sender.py:transition_state():614] send defer: 11 +2024-12-10 20:57:26,647 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,647 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 20:57:26,648 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 20:57:26,648 INFO SenderThread:598427 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 12 +2024-12-10 20:57:26,648 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,648 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 20:57:26,648 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,648 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 20:57:26,648 INFO SenderThread:598427 [file_stream.py:finish():601] file stream finish called +2024-12-10 20:57:26,855 INFO SenderThread:598427 [file_stream.py:finish():605] file stream finish is done +2024-12-10 20:57:26,855 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 13 +2024-12-10 20:57:26,855 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,855 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:transition_state():614] send defer: 14 +2024-12-10 20:57:26,856 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: defer +2024-12-10 20:57:26,856 INFO HandlerThread:598427 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send():379] send: final +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send():379] send: footer +2024-12-10 20:57:26,856 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: defer +2024-12-10 20:57:26,856 INFO SenderThread:598427 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,857 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 20:57:26,857 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: server_info +2024-12-10 20:57:26,857 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 20:57:26,859 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 20:57:26,913 DEBUG SenderThread:598427 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 20:57:26,913 INFO MainThread:598427 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 20:57:26,913 DEBUG HandlerThread:598427 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 20:57:26,914 INFO HandlerThread:598427 [handler.py:finish():882] shutting down handler +2024-12-10 20:57:27,859 INFO WriterThread:598427 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb +2024-12-10 20:57:27,913 INFO SenderThread:598427 [sender.py:finish():1615] shutting down sender +2024-12-10 20:57:27,913 INFO SenderThread:598427 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 20:57:27,913 INFO SenderThread:598427 [file_pusher.py:join():175] waiting for file pusher diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug.log b/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..39fe8b9c15afd5f65751757771ee29f6c7b8960f --- /dev/null +++ b/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Configure stats pid to 598305 +2024-12-10 20:54:57,838 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/settings +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug.log +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/wandb/run-20241210_205457-52qyo5rz/logs/debug-internal.log +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():569] calling init triggers +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():619] starting backend +2024-12-10 20:54:57,839 INFO MainThread:598305 [wandb_init.py:init():623] setting up manager +2024-12-10 20:54:57,843 INFO MainThread:598305 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 20:54:57,844 INFO MainThread:598305 [wandb_init.py:init():631] backend started and connected +2024-12-10 20:54:57,856 INFO MainThread:598305 [wandb_init.py:init():720] updated telemetry +2024-12-10 20:54:57,866 INFO MainThread:598305 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 20:54:58,163 INFO MainThread:598305 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 20:54:58,232 INFO MainThread:598305 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 20:54:58,233 INFO MainThread:598305 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 20:54:58,563 INFO MainThread:598305 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 20:54:58,565 INFO MainThread:598305 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 20:54:58,565 INFO MainThread:598305 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205451/tb, True +2024-12-10 20:55:02,121 INFO MainThread:598305 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 20:57:27,914 WARNING MsgRouterThr:598305 [router.py:message_loop():77] message_loop has been closed diff --git a/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb b/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..57b51cf2156a24653a0f7848593dc1b3ad7e4d74 Binary files /dev/null and b/205451/wandb/run-20241210_205457-52qyo5rz/run-52qyo5rz.wandb differ diff --git a/214029/.hydra/config.yaml b/214029/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/214029/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/214029/.hydra/hydra.yaml b/214029/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f95f1b3ca582ec165b7f81cc0b4636b825d3681 --- /dev/null +++ b/214029/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214029 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/214029/.hydra/overrides.yaml b/214029/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/214029/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/214029/eval_policy.log b/214029/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..de55ac5fd61943da718e607fcae626a7daabf07c --- /dev/null +++ b/214029/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-10 21:40:29,667][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 21:40:29,670][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-10 21:40:33,778][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/214032/.hydra/config.yaml b/214032/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/214032/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/214032/.hydra/hydra.yaml b/214032/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0194e72beb2cb9cbffb73fc87fc190177eab468e --- /dev/null +++ b/214032/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/214032/.hydra/overrides.yaml b/214032/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/214032/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/214032/episode_rosbags/aligned_depth_to_color_K.npy b/214032/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/214032/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/214032/episode_rosbags/cam_tf_world.npy b/214032/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/214032/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/214032/episode_rosbags/color_K.npy b/214032/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/214032/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/214032/episode_rosbags/depth_K.npy b/214032/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/214032/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/214032/episode_rosbags/episode_0_2024-12-10-21-41-00.rrd b/214032/episode_rosbags/episode_0_2024-12-10-21-41-00.rrd new file mode 100644 index 0000000000000000000000000000000000000000..a39084cc406029cbde67ae11e084d11f40336ea5 --- /dev/null +++ b/214032/episode_rosbags/episode_0_2024-12-10-21-41-00.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76893f55198fbb52e01728970c2880817c871dded8cce956bc6ff664b4769a98 +size 433299780 diff --git a/214032/episode_rosbags/episode_1_2024-12-10-21-41-44.rrd b/214032/episode_rosbags/episode_1_2024-12-10-21-41-44.rrd new file mode 100644 index 0000000000000000000000000000000000000000..a62c7422e1157a005767e61b519c494e901f7e9b --- /dev/null +++ b/214032/episode_rosbags/episode_1_2024-12-10-21-41-44.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f71284945e70d1bcd1307220f2948f95ad3c0b701a70803ebc8ad8595a9ea37 +size 433535847 diff --git a/214032/episode_rosbags/episode_2_2024-12-10-21-42-51.rrd b/214032/episode_rosbags/episode_2_2024-12-10-21-42-51.rrd new file mode 100644 index 0000000000000000000000000000000000000000..a488fabbbdc468f005031af7509104752af75d78 --- /dev/null +++ b/214032/episode_rosbags/episode_2_2024-12-10-21-42-51.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a417110e48ca05af3dbd5b814e401b46fa02e0d89f5a3c84851ad47f5d235d +size 429095900 diff --git a/214032/episode_rosbags/episode_4_2024-12-10-21-44-14.rrd b/214032/episode_rosbags/episode_4_2024-12-10-21-44-14.rrd new file mode 100644 index 0000000000000000000000000000000000000000..b7ad81b979953c34d1b30fb0e9edbf1ed3b7b0d3 --- /dev/null +++ b/214032/episode_rosbags/episode_4_2024-12-10-21-44-14.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:065eae7aeacd8bd1369e239e7b490ca6fa696e080e95f6471d9d859d453ab154 +size 432459807 diff --git a/214032/eval_robot.log b/214032/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bd6ef501d25ce8fea8c460e64ec4cc6cc2eda603 --- /dev/null +++ b/214032/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 21:40:32,383][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 21:40:32,386][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/214032/eval_video/0_eval.mp4 b/214032/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98e6b58f43c51e1099649e67d93f2923f53c87a6 --- /dev/null +++ b/214032/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cde89795fdf8f2318ab5a5df6c7470a8c3f27c70506325712d1f1ae63acf0c +size 1153889 diff --git a/214032/eval_video/1_eval.mp4 b/214032/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9074fb083583472e5edfc5f65cf0177dc9751e67 --- /dev/null +++ b/214032/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1a83ed932af73a96be91a936459619378d8947c1ececae957ffdb603897874 +size 1157518 diff --git a/214032/eval_video/2_eval.mp4 b/214032/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..941309bc399f159ff8504afa24233ba6c9a25e81 --- /dev/null +++ b/214032/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac838f24394620216df297962eea9126f794122e4334824543af281f264ae29 +size 1152465 diff --git a/214032/eval_video/3_eval.mp4 b/214032/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97c530dbfa605bde088c70c11fb1d721b1e1aeb9 --- /dev/null +++ b/214032/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3ddf93c395db4f718a9dd95be79aaf7a24dbe1b0621ffe2e060dbef9f228a1 +size 1212773 diff --git a/214032/eval_video/4_eval.mp4 b/214032/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66616bd98170a84dfd240135b401638e5c8589a6 --- /dev/null +++ b/214032/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86ec182da7a62ff7b578325c5e5e3f28bb6d9f7cc3d236c696083a8630fa2ae1 +size 1194589 diff --git a/214032/tb/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 b/214032/tb/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 new file mode 100644 index 0000000000000000000000000000000000000000..cc4c8490e7b286c109f8547d9e3d3248a5670eb8 --- /dev/null +++ b/214032/tb/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409c0b191ca76e0e9d8333b9b5747b7d2d866319845d85c75bf9c40008c5bab6 +size 1103 diff --git a/214032/wandb/debug-internal.log b/214032/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2794ac313f302d2e118d21995b461bc2b5b89a68 --- /dev/null +++ b/214032/wandb/debug-internal.log @@ -0,0 +1,749 @@ +2024-12-10 21:40:38,452 INFO StreamThr :608881 [internal.py:wandb_internal():85] W&B internal server running at pid: 608881, started at: 2024-12-10 21:40:38.452460 +2024-12-10 21:40:38,453 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:40:38,464 INFO WriterThread:608881 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb +2024-12-10 21:40:38,465 DEBUG SenderThread:608881 [sender.py:send():379] send: header +2024-12-10 21:40:38,473 DEBUG SenderThread:608881 [sender.py:send():379] send: run +2024-12-10 21:40:38,722 INFO SenderThread:608881 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files +2024-12-10 21:40:38,722 INFO SenderThread:608881 [sender.py:_start_run_threads():1188] run started: s9hpsjf9 with start time 1733884838.452824 +2024-12-10 21:40:38,737 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:40:38,737 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:40:38,787 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:40:38,792 DEBUG HandlerThread:608881 [system_info.py:__init__():26] System info init +2024-12-10 21:40:38,792 DEBUG HandlerThread:608881 [system_info.py:__init__():41] System info init done +2024-12-10 21:40:38,792 INFO HandlerThread:608881 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:40:38,792 INFO SystemMonitor:608881 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:40:38,792 INFO HandlerThread:608881 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:40:38,793 INFO SystemMonitor:608881 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:40:38,793 INFO SystemMonitor:608881 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:40:38,794 INFO SystemMonitor:608881 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:40:38,794 INFO SystemMonitor:608881 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:40:38,795 INFO SystemMonitor:608881 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:40:38,805 DEBUG HandlerThread:608881 [system_info.py:probe():152] Probing system +2024-12-10 21:40:38,807 DEBUG HandlerThread:608881 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_info.py:probe():200] Probing system done +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:40:38.805785', 'startedAt': '2024-12-11T02:40:38.443244', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3754.252, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4503.722, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 402.6240234375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:40:38,812 INFO HandlerThread:608881 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:40:38,812 INFO HandlerThread:608881 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:40:38,812 DEBUG HandlerThread:608881 [system_info.py:_save_code():44] Saving code +2024-12-10 21:40:38,813 DEBUG HandlerThread:608881 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:40:38,813 DEBUG HandlerThread:608881 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:40:38,875 DEBUG HandlerThread:608881 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:40:38,875 INFO HandlerThread:608881 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:40:38,878 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:40:38,878 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:40:38,878 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:40:38,879 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:40:39,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:40:39,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:40:39,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:40:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:40:39,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:39,154 INFO HandlerThread:608881 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/tb" + save: true +} +control { + relay_id: "59e8947623744db792dc242bf1b20464" +} +_info { + stream_id: "s9hpsjf9" +} + +2024-12-10 21:40:39,154 DEBUG HandlerThread:608881 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:40:39,157 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/r8eyuers-code/FISH/eval_robot.py +2024-12-10 21:40:39,170 INFO wandb-upload_0:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/aqm9dddc-wandb-metadata.json +2024-12-10 21:40:39,186 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/h7dhr8h6-diff.patch +2024-12-10 21:40:39,187 DEBUG SenderThread:608881 [sender.py:send():379] send: telemetry +2024-12-10 21:40:39,204 DEBUG SenderThread:608881 [sender.py:send():379] send: tbrecord +2024-12-10 21:40:39,205 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:40:39,206 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 with policy live +2024-12-10 21:40:39,437 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/0eq0y07d-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:40:39,725 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH +2024-12-10 21:40:39,725 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code +2024-12-10 21:40:40,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:41,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:41,723 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:42,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:43,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:43,447 DEBUG SenderThread:608881 [sender.py:send():379] send: config +2024-12-10 21:40:43,490 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:44,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:45,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:45,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:46,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:47,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:48,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:49,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:49,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:49,726 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:50,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:51,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:52,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:53,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:53,727 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:54,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:40:54,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:40:54,149 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:54,189 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:55,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:56,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:57,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:58,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:59,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:59,728 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:00,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:00,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:01,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:02,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:03,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:03,729 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:04,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:05,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:05,188 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:05,730 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:06,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:07,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:07,731 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:08,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:09,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:09,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:09,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:09,731 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:10,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:10,298 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:10,732 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml +2024-12-10 21:41:11,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:11,732 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:12,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:13,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:13,733 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:14,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:15,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:15,494 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:15,733 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:16,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:17,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:17,734 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:18,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:19,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:19,734 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:20,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:20,513 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:21,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:21,735 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:22,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:23,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:23,735 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:24,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:24,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:24,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:25,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:25,707 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:25,736 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:26,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:27,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:27,737 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:28,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:29,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:29,561 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:41:29,561 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 with policy now +2024-12-10 21:41:29,561 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:41:29,562 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:41:29,562 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:41:29,563 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media +2024-12-10 21:41:29,827 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/35bhfxg3-media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:30,108 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/oa60o3l4-media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:30,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:31,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:31,567 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:31,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:32,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:33,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:33,739 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:34,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:35,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:36,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:37,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:37,122 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:37,740 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:38,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:38,795 DEBUG SystemMonitor:608881 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:41:38,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:41:39,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:39,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:39,149 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:40,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:41,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:41,742 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:41:42,003 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/mz351q7u-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:41:42,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:42,229 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:43,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:43,743 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:44,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:45,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:46,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:47,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:47,234 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:47,744 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:48,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:49,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:49,745 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:50,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:51,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:51,745 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:52,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:52,431 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:53,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:53,746 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:54,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:54,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:54,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:55,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:55,747 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:56,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:57,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:57,451 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:57,748 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:58,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:59,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:59,748 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:00,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:01,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:01,749 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:02,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:02,646 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:03,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:03,750 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:04,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:05,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:05,750 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:06,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:07,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:07,751 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:07,837 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:08,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:08,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:42:09,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:09,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:09,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:09,752 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:10,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:11,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:11,752 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:12,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:13,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:13,257 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:13,753 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:13,856 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:42:13,856 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 with policy now +2024-12-10 21:42:13,859 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:42:13,859 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:42:13,859 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:42:13,860 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:42:14,124 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:14,131 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/rsee72h5-media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:14,753 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:42:14,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:14,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:42:15,088 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/4wjshnhu-media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:15,124 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:15,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:16,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:16,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:17,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:17,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:18,125 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:18,926 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:19,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:20,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:21,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:21,756 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:22,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:23,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:24,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:24,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:24,150 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:24,263 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:25,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:26,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:27,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:28,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:29,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:30,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:30,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:31,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:32,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:33,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:34,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:35,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:35,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:36,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:37,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:38,132 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:38,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:42:39,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:39,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:40,131 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:40,246 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:41,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:42,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:43,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:44,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:45,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:46,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:46,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:47,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:48,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:48,612 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:42:48,612 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:42:48,614 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:42:48,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:42:48,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:42:49,018 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/dfwzg48s-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:42:49,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:49,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:50,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:51,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:51,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:52,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:53,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:53,766 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:54,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:54,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:54,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:55,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:55,767 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:56,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:56,203 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:57,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:57,767 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:58,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:59,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:59,768 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:00,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:01,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:01,399 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:01,769 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:02,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:03,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:03,770 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:04,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:05,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:05,770 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:06,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:06,439 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:07,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:07,771 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:08,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:08,797 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:43:09,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:09,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:09,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:09,772 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:10,138 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:11,138 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:11,596 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:11,772 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:12,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:13,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:13,773 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:14,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:15,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:15,774 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:16,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:16,612 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:17,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:17,775 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:18,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:19,140 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:19,775 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:20,142 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:20,417 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:43:20,418 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:43:20,418 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_aac838f24394620216df.mp4 with policy now +2024-12-10 21:43:20,419 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:43:20,419 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:43:20,419 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:43:20,700 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/srqd6xbw-media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:43:21,087 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/739rtuu0-media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:21,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:21,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:22,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:22,435 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:22,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:23,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:24,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:24,117 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:24,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:25,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:25,777 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:26,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:27,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:27,778 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:28,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:28,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:29,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:29,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:43:30,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:30,444 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:43:30,444 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:43:30,445 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:43:30,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:43:31,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:31,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:32,153 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:33,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:33,402 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:34,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:35,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:35,781 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:36,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:37,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:37,781 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:38,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:38,516 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:38,797 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:43:39,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:39,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:39,782 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:40,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:41,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:41,783 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:42,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:43,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:43,723 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:43,784 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:44,181 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:45,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:45,785 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:46,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:47,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:47,785 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:48,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:48,919 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:49,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:49,786 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:50,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:51,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:51,787 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:52,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:53,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:53,787 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:53,921 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:54,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:54,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:54,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:55,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:55,788 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:56,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:57,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:57,789 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:58,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:59,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:59,157 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:59,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:00,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:01,151 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:01,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:02,150 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:02,301 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:44:02,302 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:44:02,302 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 with policy now +2024-12-10 21:44:02,302 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:02,303 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:02,303 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:02,586 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/by1wmb7v-media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:02,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:02,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:02,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:44:03,110 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/nincuu1n-media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:03,155 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:03,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:04,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:04,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:04,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:05,181 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:05,792 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:06,160 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:07,170 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:08,158 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:08,798 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:09,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:09,111 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:09,161 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:09,264 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:09,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:10,162 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:10,489 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:10,489 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:10,490 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:10,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:10,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:11,073 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/mo50gl5v-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:11,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:11,794 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:12,174 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:13,165 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:13,794 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:14,185 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:14,343 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:15,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:15,795 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:16,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:17,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:17,796 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:18,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:19,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:19,437 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:19,797 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:20,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:21,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:21,797 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:22,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:23,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:23,798 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:24,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:24,110 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:24,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:24,627 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:25,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:25,799 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:26,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:27,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:27,800 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:28,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:29,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:29,647 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:29,800 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:30,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:31,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:31,801 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:32,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:33,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:33,802 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:34,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:34,663 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:35,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:35,803 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:36,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:37,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:37,803 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:38,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:38,798 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:39,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:39,113 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:39,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:39,669 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:39,804 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:40,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:41,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:41,805 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:42,172 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:43,177 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:43,249 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:44:43,249 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 with policy now +2024-12-10 21:44:43,251 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:44:43,252 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:43,252 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:43,252 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:43,556 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/09qsiei9-media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:44:44,132 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/e8z9uoxa-media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:44,176 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:44,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:45,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:45,179 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:46,185 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:47,191 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:47,807 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:48,183 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:49,187 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:49,808 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:50,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:50,188 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:50,962 DEBUG SenderThread:608881 [sender.py:send():379] send: exit +2024-12-10 21:44:50,962 INFO SenderThread:608881 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:44:50,962 INFO SenderThread:608881 [sender.py:send_exit():588] handling runtime: 252 +2024-12-10 21:44:50,965 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:50,965 INFO SenderThread:608881 [sender.py:send_exit():594] send defer +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:44:50,966 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:44:50,966 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:44:50,967 DEBUG SystemMonitor:608881 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:44:50,967 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:44:50,967 DEBUG SystemMonitor:608881 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:51,016 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:51,016 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:44:51,016 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:51,016 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:44:51,808 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:51,809 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:51,809 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:57,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:57,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,019 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:44:57,019 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:44:57,019 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:57,019 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:57,019 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:44:57,020 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,024 DEBUG SenderThread:608881 [sender.py:send():379] send: summary +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,024 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:44:57,024 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,024 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:44:57,025 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:44:57,025 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:57,025 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,025 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:44:57,025 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:44:57,810 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:57,963 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:57,990 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:44:57,990 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:57,990 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,991 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:44:57,991 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,991 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:44:57,991 INFO SenderThread:608881 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:44:57,993 INFO SenderThread:608881 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:44:58,007 INFO SenderThread:608881 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:44:58,022 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:44:58,023 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:58,023 DEBUG SenderThread:608881 [sender.py:send():379] send: artifact +2024-12-10 21:44:58,023 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:44:58,597 INFO wandb-upload_0:608881 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5dl4h77g +2024-12-10 21:44:58,597 INFO wandb-upload_2:608881 [upload_job.py:push():85] Skipped uploading /tmp/tmpob1p2z0k/wandb-job.json +2024-12-10 21:44:58,811 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:58,861 INFO wandb-upload_1:608881 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjyl59zur +2024-12-10 21:44:58,963 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:59,526 INFO SenderThread:608881 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NjgxMjE3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NjgwNjQxOA==', 'versionIndex': 343}}} +2024-12-10 21:44:59,526 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:59,526 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:44:59,526 INFO SenderThread:608881 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:44:59,811 INFO SenderThread:608881 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml config.yaml +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json wandb-summary.json +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt requirements.txt +2024-12-10 21:44:59,813 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:44:59,815 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:59,816 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log output.log +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch diff.patch +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:59,818 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:44:59,820 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:59,820 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:59,820 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:44:59,820 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:59,820 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:44:59,964 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:59,964 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,000 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt +2024-12-10 21:45:00,030 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:45:00,058 INFO wandb-upload_0:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml +2024-12-10 21:45:00,095 INFO wandb-upload_4:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:45:00,103 INFO wandb-upload_3:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/j6opz80u-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:45:00,303 INFO Thread-11 (_thread_body):608881 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:45:00,303 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,303 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:45:00,304 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,304 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:45:00,304 INFO SenderThread:608881 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:45:00,305 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:45:00,305 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,305 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:45:00,305 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,305 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:45:00,305 INFO SenderThread:608881 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:45:00,501 INFO SenderThread:608881 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:45:00,502 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,502 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:45:00,502 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:45:00,502 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,502 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:45:00,502 DEBUG SenderThread:608881 [sender.py:send():379] send: final +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send():379] send: footer +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,503 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:45:00,503 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,503 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:45:00,506 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:45:00,560 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:45:00,560 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:45:00,560 INFO HandlerThread:608881 [handler.py:finish():882] shutting down handler +2024-12-10 21:45:01,505 INFO WriterThread:608881 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb +2024-12-10 21:45:01,560 INFO SenderThread:608881 [sender.py:finish():1615] shutting down sender +2024-12-10 21:45:01,560 INFO SenderThread:608881 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:45:01,560 INFO SenderThread:608881 [file_pusher.py:join():175] waiting for file pusher diff --git a/214032/wandb/debug.log b/214032/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c03fbf1338eb8bfef5e144c1346b71a6b1d55224 --- /dev/null +++ b/214032/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Configure stats pid to 608694 +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/settings +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug.log +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug-internal.log +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:40:38,446 INFO MainThread:608694 [wandb_init.py:init():619] starting backend +2024-12-10 21:40:38,446 INFO MainThread:608694 [wandb_init.py:init():623] setting up manager +2024-12-10 21:40:38,451 INFO MainThread:608694 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:40:38,452 INFO MainThread:608694 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:40:38,464 INFO MainThread:608694 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:40:38,471 INFO MainThread:608694 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:40:38,736 INFO MainThread:608694 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:40:38,778 INFO MainThread:608694 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:40:38,780 INFO MainThread:608694 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:40:39,108 INFO MainThread:608694 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:40:39,109 INFO MainThread:608694 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:40:39,110 INFO MainThread:608694 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/tb, True +2024-12-10 21:40:43,446 INFO MainThread:608694 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:45:01,564 WARNING MsgRouterThr:608694 [router.py:message_loop():77] message_loop has been closed diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py b/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml b/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f8af76f930fa3580bcc788a75a4ab7eab88109b --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733884838 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch b/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..b19c799205bed0154162a4a89b17e5e42d51ea33 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch @@ -0,0 +1,61 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..4a25ec8 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,16 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history seed 0 ++# wandb_run_id: "265638" # RGBD+Mask+act history seed 1 ++# wandb_run_id: "265637" # RGBD+Mask+act history seed 2 ++# wandb_run_id: "265636" # RGBD+Mask+act history seed 3 ++# wandb_run_id: "265635" # RGBD+Mask+act history seed 4 ++ ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +145,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos +@@ -151,7 +161,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history + # wandb_run_id: '250380_0' # RGBD+mask+contact+act history + # wandb_run_id: '250370_0' # RGBD+contact+act history + +-true_action_history: true ++true_action_history: false + wandb_notes: null + + checkpoint_epoch: 12000 # for depth + all contact +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 26f8ae3..e3031af 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -176,7 +176,7 @@ class Plan: + return interpolated_action_history + + def get_action_plan_timestamps(self): +- start_timestamp = self.ros_msg_plan.obs_header.stamp.to_sec() ++ start_timestamp = self.ros_msg_plan.obs_anchor_header.stamp.to_sec() + action_plan_timestamps = np.arange(self.trajectory_length)*self.ros_msg_plan.action_period + start_timestamp + return action_plan_timestamps + +@@ -627,7 +627,7 @@ class Franka: + # # append the gripper actions to the end of the 6d actions + # plan_dict['action_trajectory'] = np.concatenate((plan_dict['action_trajectory'], np.array(plan.gripper_action_trajectory).astype(np.float32)), axis=1) + # plan_dict['action_period'] = plan.action_period +- self.plan_queue.add(plan.header, plan.obs_header.stamp, plan_dataclass) ++ self.plan_queue.add(plan.header, plan.obs_anchor_header.stamp, plan_dataclass) + else: + rospy.logwarn('In process of resetting, ignoring plan') + pass diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..98e6b58f43c51e1099649e67d93f2923f53c87a6 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cde89795fdf8f2318ab5a5df6c7470a8c3f27c70506325712d1f1ae63acf0c +size 1153889 diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9074fb083583472e5edfc5f65cf0177dc9751e67 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1a83ed932af73a96be91a936459619378d8947c1ececae957ffdb603897874 +size 1157518 diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..941309bc399f159ff8504afa24233ba6c9a25e81 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac838f24394620216df297962eea9126f794122e4334824543af281f264ae29 +size 1152465 diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97c530dbfa605bde088c70c11fb1d721b1e1aeb9 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3ddf93c395db4f718a9dd95be79aaf7a24dbe1b0621ffe2e060dbef9f228a1 +size 1212773 diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..66616bd98170a84dfd240135b401638e5c8589a6 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86ec182da7a62ff7b578325c5e5e3f28bb6d9f7cc3d236c696083a8630fa2ae1 +size 1194589 diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log b/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e049a357b99476ddf2eb1fb39997626bd06f32a9 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log @@ -0,0 +1,6339 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733884843.447374]: resetting environment +[INFO] [1733884843.450805]: cleared current plan +[INFO] [1733884843.451672]: moving to home +[INFO] [1733884844.454048]: reached home +[INFO] [1733884844.455286]: reset action history +[INFO] [1733884847.162986]: environment reset +Starting episode 0 +[INFO] [1733884847.165016]: resetting environment +[INFO] [1733884847.171757]: cleared current plan +[INFO] [1733884847.172647]: moving to home +[INFO] [1733884848.176233]: reached home +[INFO] [1733884848.177699]: reset action history +[INFO] [1733884850.886486]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733884857.206618]: resetting environment +[INFO] [1733884857.211716]: cleared current plan +[INFO] [1733884857.212659]: moving to home +[INFO] [1733884858.214311]: reached home +[INFO] [1733884858.215785]: reset action history +[INFO] [1733884860.924842]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/episode_rosbags/episode_0_2024-12-10-21-41-00.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-608694.log +started roslaunch server http://158.130.50.37:36367/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [609002] +process[print_text-2]: started with pid [609003] +process[rosbag_record-3]: started with pid [609027] +started bagging! +[INFO] [1733884861.705117]: received mask_obs with header seq: 3662 +stamp: + secs: 1733884861 + nsecs: 708671569 +frame_id: '' and grasped_object_mask header seq: 95747 +stamp: + secs: 1733884861 + nsecs: 473592043 +frame_id: "camera_color_optical_frame" +[INFO] [1733884861.877125]: received mask_obs with header seq: 3663 +stamp: + secs: 1733884861 + nsecs: 881189584 +frame_id: '' and grasped_object_mask header seq: 95753 +stamp: + secs: 1733884861 + nsecs: 673451662 +frame_id: "camera_color_optical_frame" +[INFO] [1733884862.273033]: received mask_obs with header seq: 3664 +stamp: + secs: 1733884862 + nsecs: 277053594 +frame_id: '' and grasped_object_mask header seq: 95765 +stamp: + secs: 1733884862 + nsecs: 73149681 +frame_id: "camera_color_optical_frame" +[INFO] [1733884862.446774]: received mask_obs with header seq: 3665 +stamp: + secs: 1733884862 + nsecs: 450909614 +frame_id: '' and grasped_object_mask header seq: 95770 +stamp: + secs: 1733884862 + nsecs: 239685297 +frame_id: "camera_color_optical_frame" +[INFO] [1733884862.627665]: received mask_obs with header seq: 3666 +stamp: + secs: 1733884862 + nsecs: 631154775 +frame_id: '' and grasped_object_mask header seq: 95775 +stamp: + secs: 1733884862 + nsecs: 406220198 +frame_id: "camera_color_optical_frame" +[INFO] [1733884862.813465]: received mask_obs with header seq: 3667 +stamp: + secs: 1733884862 + nsecs: 817547321 +frame_id: '' and grasped_object_mask header seq: 95781 +stamp: + secs: 1733884862 + nsecs: 606062651 +frame_id: "camera_color_optical_frame" +[INFO] [1733884863.009212]: received mask_obs with header seq: 3668 +stamp: + secs: 1733884863 + nsecs: 13118982 +frame_id: '' and grasped_object_mask header seq: 95786 +stamp: + secs: 1733884862 + nsecs: 772599220 +frame_id: "camera_color_optical_frame" +[INFO] [1733884863.209927]: received mask_obs with header seq: 3669 +stamp: + secs: 1733884863 + nsecs: 213955640 +frame_id: '' and grasped_object_mask header seq: 95792 +stamp: + secs: 1733884862 + nsecs: 972439528 +frame_id: "camera_color_optical_frame" +[INFO] [1733884863.403934]: received mask_obs with header seq: 3670 +stamp: + secs: 1733884863 + nsecs: 407829284 +frame_id: '' and grasped_object_mask header seq: 95798 +stamp: + secs: 1733884863 + nsecs: 172278881 +frame_id: "camera_color_optical_frame" +[INFO] [1733884863.597110]: received mask_obs with header seq: 3671 +stamp: + secs: 1733884863 + nsecs: 601004123 +frame_id: '' and grasped_object_mask header seq: 95804 +stamp: + secs: 1733884863 + nsecs: 372118235 +frame_id: "camera_color_optical_frame" +[INFO] [1733884863.802798]: received mask_obs with header seq: 3672 +stamp: + secs: 1733884863 + nsecs: 806773185 +frame_id: '' and grasped_object_mask header seq: 95810 +stamp: + secs: 1733884863 + nsecs: 571958542 +frame_id: "camera_color_optical_frame" +[INFO] [1733884864.008041]: received mask_obs with header seq: 3673 +stamp: + secs: 1733884864 + nsecs: 12117147 +frame_id: '' and grasped_object_mask header seq: 95816 +stamp: + secs: 1733884863 + nsecs: 771797895 +frame_id: "camera_color_optical_frame" +[INFO] [1733884864.214759]: received mask_obs with header seq: 3674 +stamp: + secs: 1733884864 + nsecs: 218813180 +frame_id: '' and grasped_object_mask header seq: 95822 +stamp: + secs: 1733884863 + nsecs: 971632481 +frame_id: "camera_color_optical_frame" +[INFO] [1733884864.411858]: received mask_obs with header seq: 3675 +stamp: + secs: 1733884864 + nsecs: 415646791 +frame_id: '' and grasped_object_mask header seq: 95828 +stamp: + secs: 1733884864 + nsecs: 171464682 +frame_id: "camera_color_optical_frame" +[INFO] [1733884864.604551]: received mask_obs with header seq: 3676 +stamp: + secs: 1733884864 + nsecs: 608528137 +frame_id: '' and grasped_object_mask header seq: 95834 +stamp: + secs: 1733884864 + nsecs: 371296644 +frame_id: "camera_color_optical_frame" +[INFO] [1733884864.799896]: received mask_obs with header seq: 3677 +stamp: + secs: 1733884864 + nsecs: 803970336 +frame_id: '' and grasped_object_mask header seq: 95840 +stamp: + secs: 1733884864 + nsecs: 571128130 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.003313]: received mask_obs with header seq: 3678 +stamp: + secs: 1733884865 + nsecs: 7215976 +frame_id: '' and grasped_object_mask header seq: 95846 +stamp: + secs: 1733884864 + nsecs: 770960808 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.187775]: received mask_obs with header seq: 3679 +stamp: + secs: 1733884865 + nsecs: 190089941 +frame_id: '' and grasped_object_mask header seq: 95852 +stamp: + secs: 1733884864 + nsecs: 970795870 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.390949]: received mask_obs with header seq: 3680 +stamp: + secs: 1733884865 + nsecs: 394893884 +frame_id: '' and grasped_object_mask header seq: 95858 +stamp: + secs: 1733884865 + nsecs: 170633554 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.587151]: received mask_obs with header seq: 3681 +stamp: + secs: 1733884865 + nsecs: 588657855 +frame_id: '' and grasped_object_mask header seq: 95864 +stamp: + secs: 1733884865 + nsecs: 370471001 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.781564]: received mask_obs with header seq: 3682 +stamp: + secs: 1733884865 + nsecs: 785491704 +frame_id: '' and grasped_object_mask header seq: 95869 +stamp: + secs: 1733884865 + nsecs: 537002802 +frame_id: "camera_color_optical_frame" +[INFO] [1733884865.969831]: received mask_obs with header seq: 3683 +stamp: + secs: 1733884865 + nsecs: 973886251 +frame_id: '' and grasped_object_mask header seq: 95875 +stamp: + secs: 1733884865 + nsecs: 736840010 +frame_id: "camera_color_optical_frame" +[INFO] [1733884866.168761]: received mask_obs with header seq: 3684 +stamp: + secs: 1733884866 + nsecs: 172855138 +frame_id: '' and grasped_object_mask header seq: 95881 +stamp: + secs: 1733884865 + nsecs: 936680317 +frame_id: "camera_color_optical_frame" +[INFO] [1733884866.368024]: received mask_obs with header seq: 3685 +stamp: + secs: 1733884866 + nsecs: 372026681 +frame_id: '' and grasped_object_mask header seq: 95887 +stamp: + secs: 1733884866 + nsecs: 136523247 +frame_id: "camera_color_optical_frame" +[INFO] [1733884866.571757]: received mask_obs with header seq: 3686 +stamp: + secs: 1733884866 + nsecs: 575263500 +frame_id: '' and grasped_object_mask header seq: 95893 +stamp: + secs: 1733884866 + nsecs: 336366177 +frame_id: "camera_color_optical_frame" +[INFO] [1733884866.766699]: received mask_obs with header seq: 3687 +stamp: + secs: 1733884866 + nsecs: 770708084 +frame_id: '' and grasped_object_mask header seq: 95899 +stamp: + secs: 1733884866 + nsecs: 536209345 +frame_id: "camera_color_optical_frame" +[INFO] [1733884866.957776]: received mask_obs with header seq: 3688 +stamp: + secs: 1733884866 + nsecs: 961687326 +frame_id: '' and grasped_object_mask header seq: 95905 +stamp: + secs: 1733884866 + nsecs: 736053705 +frame_id: "camera_color_optical_frame" +[INFO] [1733884867.141172]: received mask_obs with header seq: 3689 +stamp: + secs: 1733884867 + nsecs: 145236730 +frame_id: '' and grasped_object_mask header seq: 95910 +stamp: + secs: 1733884866 + nsecs: 902589083 +frame_id: "camera_color_optical_frame" +[INFO] [1733884867.345771]: received mask_obs with header seq: 3690 +stamp: + secs: 1733884867 + nsecs: 349540710 +frame_id: '' and grasped_object_mask header seq: 95916 +stamp: + secs: 1733884867 + nsecs: 102431774 +frame_id: "camera_color_optical_frame" +[INFO] [1733884867.540455]: received mask_obs with header seq: 3691 +stamp: + secs: 1733884867 + nsecs: 544493913 +frame_id: '' and grasped_object_mask header seq: 95922 +stamp: + secs: 1733884867 + nsecs: 302275896 +frame_id: "camera_color_optical_frame" +[INFO] [1733884867.742126]: received mask_obs with header seq: 3692 +stamp: + secs: 1733884867 + nsecs: 745834827 +frame_id: '' and grasped_object_mask header seq: 95928 +stamp: + secs: 1733884867 + nsecs: 502118826 +frame_id: "camera_color_optical_frame" +[INFO] [1733884867.937647]: received mask_obs with header seq: 3693 +stamp: + secs: 1733884867 + nsecs: 941672086 +frame_id: '' and grasped_object_mask header seq: 95934 +stamp: + secs: 1733884867 + nsecs: 701962233 +frame_id: "camera_color_optical_frame" +[INFO] [1733884868.138633]: received mask_obs with header seq: 3694 +stamp: + secs: 1733884868 + nsecs: 142326831 +frame_id: '' and grasped_object_mask header seq: 95940 +stamp: + secs: 1733884867 + nsecs: 901805878 +frame_id: "camera_color_optical_frame" +[INFO] [1733884868.325430]: received mask_obs with header seq: 3695 +stamp: + secs: 1733884868 + nsecs: 329392671 +frame_id: '' and grasped_object_mask header seq: 95946 +stamp: + secs: 1733884868 + nsecs: 101649761 +frame_id: "camera_color_optical_frame" +[INFO] [1733884868.533155]: received mask_obs with header seq: 3696 +stamp: + secs: 1733884868 + nsecs: 536687850 +frame_id: '' and grasped_object_mask header seq: 95952 +stamp: + secs: 1733884868 + nsecs: 301493645 +frame_id: "camera_color_optical_frame" +[INFO] [1733884868.722360]: received mask_obs with header seq: 3697 +stamp: + secs: 1733884868 + nsecs: 725866317 +frame_id: '' and grasped_object_mask header seq: 95958 +stamp: + secs: 1733884868 + nsecs: 501338959 +frame_id: "camera_color_optical_frame" +[INFO] [1733884868.917154]: received mask_obs with header seq: 3698 +stamp: + secs: 1733884868 + nsecs: 921156883 +frame_id: '' and grasped_object_mask header seq: 95964 +stamp: + secs: 1733884868 + nsecs: 701183081 +frame_id: "camera_color_optical_frame" +[INFO] [1733884869.114291]: received mask_obs with header seq: 3699 +stamp: + secs: 1733884869 + nsecs: 118330478 +frame_id: '' and grasped_object_mask header seq: 95970 +stamp: + secs: 1733884868 + nsecs: 901024580 +frame_id: "camera_color_optical_frame" +[INFO] [1733884869.299592]: received mask_obs with header seq: 3700 +stamp: + secs: 1733884869 + nsecs: 303411960 +frame_id: '' and grasped_object_mask header seq: 95975 +stamp: + secs: 1733884869 + nsecs: 67554951 +frame_id: "camera_color_optical_frame" +[INFO] [1733884869.506844]: received mask_obs with header seq: 3701 +stamp: + secs: 1733884869 + nsecs: 510773420 +frame_id: '' and grasped_object_mask header seq: 95981 +stamp: + secs: 1733884869 + nsecs: 267390728 +frame_id: "camera_color_optical_frame" +[INFO] [1733884869.699442]: received mask_obs with header seq: 3702 +stamp: + secs: 1733884869 + nsecs: 702786207 +frame_id: '' and grasped_object_mask header seq: 95987 +stamp: + secs: 1733884869 + nsecs: 467225790 +frame_id: "camera_color_optical_frame" +[INFO] [1733884869.891614]: received mask_obs with header seq: 3703 +stamp: + secs: 1733884869 + nsecs: 895241975 +frame_id: '' and grasped_object_mask header seq: 95993 +stamp: + secs: 1733884869 + nsecs: 667061090 +frame_id: "camera_color_optical_frame" +[INFO] [1733884870.100729]: received mask_obs with header seq: 3704 +stamp: + secs: 1733884870 + nsecs: 104778766 +frame_id: '' and grasped_object_mask header seq: 95999 +stamp: + secs: 1733884869 + nsecs: 866896868 +frame_id: "camera_color_optical_frame" +[INFO] [1733884870.294901]: received mask_obs with header seq: 3705 +stamp: + secs: 1733884870 + nsecs: 298915386 +frame_id: '' and grasped_object_mask header seq: 96005 +stamp: + secs: 1733884870 + nsecs: 66736460 +frame_id: "camera_color_optical_frame" +[INFO] [1733884870.498479]: received mask_obs with header seq: 3706 +stamp: + secs: 1733884870 + nsecs: 502567768 +frame_id: '' and grasped_object_mask header seq: 96011 +stamp: + secs: 1733884870 + nsecs: 266577244 +frame_id: "camera_color_optical_frame" +[INFO] [1733884870.701424]: received mask_obs with header seq: 3707 +stamp: + secs: 1733884870 + nsecs: 705351829 +frame_id: '' and grasped_object_mask header seq: 96017 +stamp: + secs: 1733884870 + nsecs: 466416836 +frame_id: "camera_color_optical_frame" +[INFO] [1733884870.901573]: received mask_obs with header seq: 3708 +stamp: + secs: 1733884870 + nsecs: 905528306 +frame_id: '' and grasped_object_mask header seq: 96023 +stamp: + secs: 1733884870 + nsecs: 666256428 +frame_id: "camera_color_optical_frame" +[INFO] [1733884871.092751]: received mask_obs with header seq: 3709 +stamp: + secs: 1733884871 + nsecs: 95873832 +frame_id: '' and grasped_object_mask header seq: 96029 +stamp: + secs: 1733884870 + nsecs: 866096020 +frame_id: "camera_color_optical_frame" +[INFO] [1733884871.291785]: received mask_obs with header seq: 3710 +stamp: + secs: 1733884871 + nsecs: 294553279 +frame_id: '' and grasped_object_mask header seq: 96035 +stamp: + secs: 1733884871 + nsecs: 65935850 +frame_id: "camera_color_optical_frame" +[INFO] [1733884871.500061]: received mask_obs with header seq: 3711 +stamp: + secs: 1733884871 + nsecs: 504066705 +frame_id: '' and grasped_object_mask header seq: 96041 +stamp: + secs: 1733884871 + nsecs: 265775681 +frame_id: "camera_color_optical_frame" +[INFO] [1733884871.687014]: received mask_obs with header seq: 3712 +stamp: + secs: 1733884871 + nsecs: 690725326 +frame_id: '' and grasped_object_mask header seq: 96047 +stamp: + secs: 1733884871 + nsecs: 465615511 +frame_id: "camera_color_optical_frame" +[INFO] [1733884871.891281]: received mask_obs with header seq: 3713 +stamp: + secs: 1733884871 + nsecs: 895286798 +frame_id: '' and grasped_object_mask header seq: 96053 +stamp: + secs: 1733884871 + nsecs: 665456295 +frame_id: "camera_color_optical_frame" +[INFO] [1733884872.096935]: received mask_obs with header seq: 3714 +stamp: + secs: 1733884872 + nsecs: 100926637 +frame_id: '' and grasped_object_mask header seq: 96059 +stamp: + secs: 1733884871 + nsecs: 865295887 +frame_id: "camera_color_optical_frame" +[INFO] [1733884872.297763]: received mask_obs with header seq: 3715 +stamp: + secs: 1733884872 + nsecs: 301606893 +frame_id: '' and grasped_object_mask header seq: 96065 +stamp: + secs: 1733884872 + nsecs: 65135002 +frame_id: "camera_color_optical_frame" +[INFO] [1733884872.496754]: received mask_obs with header seq: 3716 +stamp: + secs: 1733884872 + nsecs: 500658273 +frame_id: '' and grasped_object_mask header seq: 96071 +stamp: + secs: 1733884872 + nsecs: 264973879 +frame_id: "camera_color_optical_frame" +[INFO] [1733884872.689003]: received mask_obs with header seq: 3717 +stamp: + secs: 1733884872 + nsecs: 692953824 +frame_id: '' and grasped_object_mask header seq: 96077 +stamp: + secs: 1733884872 + nsecs: 464812756 +frame_id: "camera_color_optical_frame" +[INFO] [1733884872.890722]: received mask_obs with header seq: 3718 +stamp: + secs: 1733884872 + nsecs: 894833564 +frame_id: '' and grasped_object_mask header seq: 96083 +stamp: + secs: 1733884872 + nsecs: 664651632 +frame_id: "camera_color_optical_frame" +[INFO] [1733884873.095181]: received mask_obs with header seq: 3719 +stamp: + secs: 1733884873 + nsecs: 99262237 +frame_id: '' and grasped_object_mask header seq: 96089 +stamp: + secs: 1733884872 + nsecs: 864490271 +frame_id: "camera_color_optical_frame" +[INFO] [1733884873.290114]: received mask_obs with header seq: 3720 +stamp: + secs: 1733884873 + nsecs: 294124364 +frame_id: '' and grasped_object_mask header seq: 96095 +stamp: + secs: 1733884873 + nsecs: 64320803 +frame_id: "camera_color_optical_frame" +[INFO] [1733884873.496191]: received mask_obs with header seq: 3721 +stamp: + secs: 1733884873 + nsecs: 500174283 +frame_id: '' and grasped_object_mask header seq: 96101 +stamp: + secs: 1733884873 + nsecs: 264151096 +frame_id: "camera_color_optical_frame" +[INFO] [1733884873.691754]: received mask_obs with header seq: 3722 +stamp: + secs: 1733884873 + nsecs: 695552110 +frame_id: '' and grasped_object_mask header seq: 96107 +stamp: + secs: 1733884873 + nsecs: 463980913 +frame_id: "camera_color_optical_frame" +[INFO] [1733884873.890146]: received mask_obs with header seq: 3723 +stamp: + secs: 1733884873 + nsecs: 894157171 +frame_id: '' and grasped_object_mask header seq: 96113 +stamp: + secs: 1733884873 + nsecs: 663810492 +frame_id: "camera_color_optical_frame" +[INFO] [1733884874.095499]: received mask_obs with header seq: 3724 +stamp: + secs: 1733884874 + nsecs: 99298715 +frame_id: '' and grasped_object_mask header seq: 96119 +stamp: + secs: 1733884873 + nsecs: 863640547 +frame_id: "camera_color_optical_frame" +[INFO] [1733884874.294636]: received mask_obs with header seq: 3725 +stamp: + secs: 1733884874 + nsecs: 298535108 +frame_id: '' and grasped_object_mask header seq: 96125 +stamp: + secs: 1733884874 + nsecs: 63476086 +frame_id: "camera_color_optical_frame" +[INFO] [1733884874.493181]: received mask_obs with header seq: 3726 +stamp: + secs: 1733884874 + nsecs: 497219324 +frame_id: '' and grasped_object_mask header seq: 96131 +stamp: + secs: 1733884874 + nsecs: 263310194 +frame_id: "camera_color_optical_frame" +[INFO] [1733884874.693697]: received mask_obs with header seq: 3727 +stamp: + secs: 1733884874 + nsecs: 697718620 +frame_id: '' and grasped_object_mask header seq: 96137 +stamp: + secs: 1733884874 + nsecs: 463144541 +frame_id: "camera_color_optical_frame" +[INFO] [1733884874.898525]: received mask_obs with header seq: 3728 +stamp: + secs: 1733884874 + nsecs: 902585029 +frame_id: '' and grasped_object_mask header seq: 96143 +stamp: + secs: 1733884874 + nsecs: 662978411 +frame_id: "camera_color_optical_frame" +[INFO] [1733884875.093674]: received mask_obs with header seq: 3729 +stamp: + secs: 1733884875 + nsecs: 97754478 +frame_id: '' and grasped_object_mask header seq: 96149 +stamp: + secs: 1733884874 + nsecs: 862812996 +frame_id: "camera_color_optical_frame" +[INFO] [1733884875.296431]: received mask_obs with header seq: 3730 +stamp: + secs: 1733884875 + nsecs: 300483465 +frame_id: '' and grasped_object_mask header seq: 96155 +stamp: + secs: 1733884875 + nsecs: 62650442 +frame_id: "camera_color_optical_frame" +[INFO] [1733884875.494435]: received mask_obs with header seq: 3731 +stamp: + secs: 1733884875 + nsecs: 498356342 +frame_id: '' and grasped_object_mask header seq: 96161 +stamp: + secs: 1733884875 + nsecs: 262488604 +frame_id: "camera_color_optical_frame" +[INFO] [1733884875.690826]: received mask_obs with header seq: 3732 +stamp: + secs: 1733884875 + nsecs: 694913625 +frame_id: '' and grasped_object_mask header seq: 96167 +stamp: + secs: 1733884875 + nsecs: 462325811 +frame_id: "camera_color_optical_frame" +[INFO] [1733884875.896044]: received mask_obs with header seq: 3733 +stamp: + secs: 1733884875 + nsecs: 900053024 +frame_id: '' and grasped_object_mask header seq: 96173 +stamp: + secs: 1733884875 + nsecs: 662163019 +frame_id: "camera_color_optical_frame" +[INFO] [1733884876.094466]: received mask_obs with header seq: 3734 +stamp: + secs: 1733884876 + nsecs: 98357677 +frame_id: '' and grasped_object_mask header seq: 96179 +stamp: + secs: 1733884875 + nsecs: 862000227 +frame_id: "camera_color_optical_frame" +[INFO] [1733884876.293684]: received mask_obs with header seq: 3735 +stamp: + secs: 1733884876 + nsecs: 297608613 +frame_id: '' and grasped_object_mask header seq: 96185 +stamp: + secs: 1733884876 + nsecs: 61837435 +frame_id: "camera_color_optical_frame" +[INFO] [1733884876.487923]: received mask_obs with header seq: 3736 +stamp: + secs: 1733884876 + nsecs: 491653919 +frame_id: '' and grasped_object_mask header seq: 96191 +stamp: + secs: 1733884876 + nsecs: 261674404 +frame_id: "camera_color_optical_frame" +[INFO] [1733884876.689817]: received mask_obs with header seq: 3737 +stamp: + secs: 1733884876 + nsecs: 693862438 +frame_id: '' and grasped_object_mask header seq: 96197 +stamp: + secs: 1733884876 + nsecs: 461512089 +frame_id: "camera_color_optical_frame" +[INFO] [1733884876.895917]: received mask_obs with header seq: 3738 +stamp: + secs: 1733884876 + nsecs: 899998188 +frame_id: '' and grasped_object_mask header seq: 96203 +stamp: + secs: 1733884876 + nsecs: 661348820 +frame_id: "camera_color_optical_frame" +[INFO] [1733884877.090372]: received mask_obs with header seq: 3739 +stamp: + secs: 1733884877 + nsecs: 94211816 +frame_id: '' and grasped_object_mask header seq: 96209 +stamp: + secs: 1733884876 + nsecs: 861186743 +frame_id: "camera_color_optical_frame" +[INFO] [1733884877.293409]: received mask_obs with header seq: 3740 +stamp: + secs: 1733884877 + nsecs: 297040700 +frame_id: '' and grasped_object_mask header seq: 96215 +stamp: + secs: 1733884877 + nsecs: 61031580 +frame_id: "camera_color_optical_frame" +[INFO] [1733884877.495067]: received mask_obs with header seq: 3741 +stamp: + secs: 1733884877 + nsecs: 499115943 +frame_id: '' and grasped_object_mask header seq: 96221 +stamp: + secs: 1733884877 + nsecs: 260876656 +frame_id: "camera_color_optical_frame" +[INFO] [1733884877.687122]: received mask_obs with header seq: 3742 +stamp: + secs: 1733884877 + nsecs: 691226720 +frame_id: '' and grasped_object_mask header seq: 96227 +stamp: + secs: 1733884877 + nsecs: 460722208 +frame_id: "camera_color_optical_frame" +[INFO] [1733884877.894825]: received mask_obs with header seq: 3743 +stamp: + secs: 1733884877 + nsecs: 898905038 +frame_id: '' and grasped_object_mask header seq: 96233 +stamp: + secs: 1733884877 + nsecs: 660569429 +frame_id: "camera_color_optical_frame" +[INFO] [1733884878.089429]: received mask_obs with header seq: 3744 +stamp: + secs: 1733884878 + nsecs: 93441009 +frame_id: '' and grasped_object_mask header seq: 96239 +stamp: + secs: 1733884877 + nsecs: 860414743 +frame_id: "camera_color_optical_frame" +[INFO] [1733884878.290930]: received mask_obs with header seq: 3745 +stamp: + secs: 1733884878 + nsecs: 294970989 +frame_id: '' and grasped_object_mask header seq: 96245 +stamp: + secs: 1733884878 + nsecs: 60255527 +frame_id: "camera_color_optical_frame" +[INFO] [1733884878.488953]: received mask_obs with header seq: 3746 +stamp: + secs: 1733884878 + nsecs: 493054866 +frame_id: '' and grasped_object_mask header seq: 96251 +stamp: + secs: 1733884878 + nsecs: 260096073 +frame_id: "camera_color_optical_frame" +[INFO] [1733884878.689926]: received mask_obs with header seq: 3747 +stamp: + secs: 1733884878 + nsecs: 693885087 +frame_id: '' and grasped_object_mask header seq: 96257 +stamp: + secs: 1733884878 + nsecs: 459937334 +frame_id: "camera_color_optical_frame" +[INFO] [1733884878.899404]: received mask_obs with header seq: 3748 +stamp: + secs: 1733884878 + nsecs: 903481245 +frame_id: '' and grasped_object_mask header seq: 96263 +stamp: + secs: 1733884878 + nsecs: 659778118 +frame_id: "camera_color_optical_frame" +[INFO] [1733884879.088532]: received mask_obs with header seq: 3749 +stamp: + secs: 1733884879 + nsecs: 92489957 +frame_id: '' and grasped_object_mask header seq: 96269 +stamp: + secs: 1733884878 + nsecs: 859618187 +frame_id: "camera_color_optical_frame" +[INFO] [1733884879.290518]: received mask_obs with header seq: 3750 +stamp: + secs: 1733884879 + nsecs: 292949199 +frame_id: '' and grasped_object_mask header seq: 96275 +stamp: + secs: 1733884879 + nsecs: 59454203 +frame_id: "camera_color_optical_frame" +[INFO] [1733884879.490175]: received mask_obs with header seq: 3751 +stamp: + secs: 1733884879 + nsecs: 494203805 +frame_id: '' and grasped_object_mask header seq: 96281 +stamp: + secs: 1733884879 + nsecs: 259288788 +frame_id: "camera_color_optical_frame" +[INFO] [1733884879.690634]: received mask_obs with header seq: 3752 +stamp: + secs: 1733884879 + nsecs: 694404602 +frame_id: '' and grasped_object_mask header seq: 96287 +stamp: + secs: 1733884879 + nsecs: 459123373 +frame_id: "camera_color_optical_frame" +[INFO] [1733884879.890381]: received mask_obs with header seq: 3753 +stamp: + secs: 1733884879 + nsecs: 894209861 +frame_id: '' and grasped_object_mask header seq: 96293 +stamp: + secs: 1733884879 + nsecs: 658957720 +frame_id: "camera_color_optical_frame" +[INFO] [1733884880.091671]: received mask_obs with header seq: 3754 +stamp: + secs: 1733884880 + nsecs: 95725297 +frame_id: '' and grasped_object_mask header seq: 96299 +stamp: + secs: 1733884879 + nsecs: 858793020 +frame_id: "camera_color_optical_frame" +[INFO] [1733884880.291703]: received mask_obs with header seq: 3755 +stamp: + secs: 1733884880 + nsecs: 295709133 +frame_id: '' and grasped_object_mask header seq: 96305 +stamp: + secs: 1733884880 + nsecs: 58631420 +frame_id: "camera_color_optical_frame" +[INFO] [1733884880.513069]: received mask_obs with header seq: 3756 +stamp: + secs: 1733884880 + nsecs: 516759395 +frame_id: '' and grasped_object_mask header seq: 96311 +stamp: + secs: 1733884880 + nsecs: 258471012 +frame_id: "camera_color_optical_frame" +[INFO] [1733884880.719241]: received mask_obs with header seq: 3757 +stamp: + secs: 1733884880 + nsecs: 723034381 +frame_id: '' and grasped_object_mask header seq: 96317 +stamp: + secs: 1733884880 + nsecs: 458309174 +frame_id: "camera_color_optical_frame" +[INFO] [1733884880.916185]: received mask_obs with header seq: 3758 +stamp: + secs: 1733884880 + nsecs: 920284271 +frame_id: '' and grasped_object_mask header seq: 96324 +stamp: + secs: 1733884880 + nsecs: 691453218 +frame_id: "camera_color_optical_frame" +[INFO] [1733884881.116214]: received mask_obs with header seq: 3759 +stamp: + secs: 1733884881 + nsecs: 120014905 +frame_id: '' and grasped_object_mask header seq: 96330 +stamp: + secs: 1733884880 + nsecs: 891293049 +frame_id: "camera_color_optical_frame" +[INFO] [1733884881.321151]: received mask_obs with header seq: 3760 +stamp: + secs: 1733884881 + nsecs: 325087785 +frame_id: '' and grasped_object_mask header seq: 96336 +stamp: + secs: 1733884881 + nsecs: 91133595 +frame_id: "camera_color_optical_frame" +[INFO] [1733884881.525493]: received mask_obs with header seq: 3761 +stamp: + secs: 1733884881 + nsecs: 529554367 +frame_id: '' and grasped_object_mask header seq: 96342 +stamp: + secs: 1733884881 + nsecs: 290974140 +frame_id: "camera_color_optical_frame" +[INFO] [1733884881.716712]: received mask_obs with header seq: 3762 +stamp: + secs: 1733884881 + nsecs: 720673322 +frame_id: '' and grasped_object_mask header seq: 96348 +stamp: + secs: 1733884881 + nsecs: 490814686 +frame_id: "camera_color_optical_frame" +[INFO] [1733884881.928203]: received mask_obs with header seq: 3763 +stamp: + secs: 1733884881 + nsecs: 931960582 +frame_id: '' and grasped_object_mask header seq: 96354 +stamp: + secs: 1733884881 + nsecs: 690655231 +frame_id: "camera_color_optical_frame" +[INFO] [1733884882.126812]: received mask_obs with header seq: 3764 +stamp: + secs: 1733884882 + nsecs: 130887508 +frame_id: '' and grasped_object_mask header seq: 96360 +stamp: + secs: 1733884881 + nsecs: 890495777 +frame_id: "camera_color_optical_frame" +[INFO] [1733884882.320576]: received mask_obs with header seq: 3765 +stamp: + secs: 1733884882 + nsecs: 324506998 +frame_id: '' and grasped_object_mask header seq: 96366 +stamp: + secs: 1733884882 + nsecs: 90337276 +frame_id: "camera_color_optical_frame" +[INFO] [1733884882.519736]: received mask_obs with header seq: 3766 +stamp: + secs: 1733884882 + nsecs: 523744344 +frame_id: '' and grasped_object_mask header seq: 96372 +stamp: + secs: 1733884882 + nsecs: 290177822 +frame_id: "camera_color_optical_frame" +[INFO] [1733884882.716763]: received mask_obs with header seq: 3767 +stamp: + secs: 1733884882 + nsecs: 720804214 +frame_id: '' and grasped_object_mask header seq: 96378 +stamp: + secs: 1733884882 + nsecs: 490018606 +frame_id: "camera_color_optical_frame" +[INFO] [1733884882.916438]: received mask_obs with header seq: 3768 +stamp: + secs: 1733884882 + nsecs: 920472145 +frame_id: '' and grasped_object_mask header seq: 96384 +stamp: + secs: 1733884882 + nsecs: 689859390 +frame_id: "camera_color_optical_frame" +[INFO] [1733884883.110464]: received mask_obs with header seq: 3769 +stamp: + secs: 1733884883 + nsecs: 114572286 +frame_id: '' and grasped_object_mask header seq: 96390 +stamp: + secs: 1733884882 + nsecs: 889699697 +frame_id: "camera_color_optical_frame" +[INFO] [1733884883.303210]: received mask_obs with header seq: 3770 +stamp: + secs: 1733884883 + nsecs: 307040929 +frame_id: '' and grasped_object_mask header seq: 96395 +stamp: + secs: 1733884883 + nsecs: 56233168 +frame_id: "camera_color_optical_frame" +[INFO] [1733884883.498435]: received mask_obs with header seq: 3771 +stamp: + secs: 1733884883 + nsecs: 502550363 +frame_id: '' and grasped_object_mask header seq: 96402 +stamp: + secs: 1733884883 + nsecs: 289379835 +frame_id: "camera_color_optical_frame" +[INFO] [1733884883.687502]: received mask_obs with header seq: 3772 +stamp: + secs: 1733884883 + nsecs: 691600561 +frame_id: '' and grasped_object_mask header seq: 96407 +stamp: + secs: 1733884883 + nsecs: 455912352 +frame_id: "camera_color_optical_frame" +[INFO] [1733884883.894686]: received mask_obs with header seq: 3773 +stamp: + secs: 1733884883 + nsecs: 898758888 +frame_id: '' and grasped_object_mask header seq: 96413 +stamp: + secs: 1733884883 + nsecs: 655751944 +frame_id: "camera_color_optical_frame" +[INFO] [1733884884.095354]: received mask_obs with header seq: 3774 +stamp: + secs: 1733884884 + nsecs: 99329948 +frame_id: '' and grasped_object_mask header seq: 96419 +stamp: + secs: 1733884883 + nsecs: 855592966 +frame_id: "camera_color_optical_frame" +[INFO] [1733884884.288573]: received mask_obs with header seq: 3775 +stamp: + secs: 1733884884 + nsecs: 292556047 +frame_id: '' and grasped_object_mask header seq: 96425 +stamp: + secs: 1733884884 + nsecs: 55433512 +frame_id: "camera_color_optical_frame" +[INFO] [1733884884.509258]: received mask_obs with header seq: 3776 +stamp: + secs: 1733884884 + nsecs: 513336896 +frame_id: '' and grasped_object_mask header seq: 96431 +stamp: + secs: 1733884884 + nsecs: 255274773 +frame_id: "camera_color_optical_frame" +[INFO] [1733884884.724344]: received mask_obs with header seq: 3777 +stamp: + secs: 1733884884 + nsecs: 727698087 +frame_id: '' and grasped_object_mask header seq: 96438 +stamp: + secs: 1733884884 + nsecs: 488423109 +frame_id: "camera_color_optical_frame" +[INFO] [1733884884.912989]: received mask_obs with header seq: 3778 +stamp: + secs: 1733884884 + nsecs: 916564464 +frame_id: '' and grasped_object_mask header seq: 96444 +stamp: + secs: 1733884884 + nsecs: 688264370 +frame_id: "camera_color_optical_frame" +[INFO] [1733884885.107274]: received mask_obs with header seq: 3779 +stamp: + secs: 1733884885 + nsecs: 111221313 +frame_id: '' and grasped_object_mask header seq: 96450 +stamp: + secs: 1733884884 + nsecs: 888105154 +frame_id: "camera_color_optical_frame" +[INFO] [1733884885.312772]: received mask_obs with header seq: 3780 +stamp: + secs: 1733884885 + nsecs: 316696166 +frame_id: '' and grasped_object_mask header seq: 96456 +stamp: + secs: 1733884885 + nsecs: 87944508 +frame_id: "camera_color_optical_frame" +[INFO] [1733884885.511250]: received mask_obs with header seq: 3781 +stamp: + secs: 1733884885 + nsecs: 515220642 +frame_id: '' and grasped_object_mask header seq: 96462 +stamp: + secs: 1733884885 + nsecs: 287783623 +frame_id: "camera_color_optical_frame" +[INFO] [1733884885.706447]: received mask_obs with header seq: 3782 +stamp: + secs: 1733884885 + nsecs: 709109783 +frame_id: '' and grasped_object_mask header seq: 96468 +stamp: + secs: 1733884885 + nsecs: 487622976 +frame_id: "camera_color_optical_frame" +[INFO] [1733884885.899475]: received mask_obs with header seq: 3783 +stamp: + secs: 1733884885 + nsecs: 903315305 +frame_id: '' and grasped_object_mask header seq: 96473 +stamp: + secs: 1733884885 + nsecs: 654156208 +frame_id: "camera_color_optical_frame" +[INFO] [1733884886.089959]: received mask_obs with header seq: 3784 +stamp: + secs: 1733884886 + nsecs: 94031333 +frame_id: '' and grasped_object_mask header seq: 96479 +stamp: + secs: 1733884885 + nsecs: 853995323 +frame_id: "camera_color_optical_frame" +[INFO] [1733884886.301834]: received mask_obs with header seq: 3785 +stamp: + secs: 1733884886 + nsecs: 305861711 +frame_id: '' and grasped_object_mask header seq: 96485 +stamp: + secs: 1733884886 + nsecs: 53832531 +frame_id: "camera_color_optical_frame" +[INFO] [1733884886.502340]: received mask_obs with header seq: 3786 +stamp: + secs: 1733884886 + nsecs: 506267547 +frame_id: '' and grasped_object_mask header seq: 96491 +stamp: + secs: 1733884886 + nsecs: 253669739 +frame_id: "camera_color_optical_frame" +[INFO] [1733884886.701484]: received mask_obs with header seq: 3787 +stamp: + secs: 1733884886 + nsecs: 705464601 +frame_id: '' and grasped_object_mask header seq: 96497 +stamp: + secs: 1733884886 + nsecs: 453508139 +frame_id: "camera_color_optical_frame" +[INFO] [1733884886.909770]: received mask_obs with header seq: 3788 +stamp: + secs: 1733884886 + nsecs: 913884401 +frame_id: '' and grasped_object_mask header seq: 96503 +stamp: + secs: 1733884886 + nsecs: 653345108 +frame_id: "camera_color_optical_frame" +[INFO] [1733884887.110428]: received mask_obs with header seq: 3789 +stamp: + secs: 1733884887 + nsecs: 114512443 +frame_id: '' and grasped_object_mask header seq: 96510 +stamp: + secs: 1733884886 + nsecs: 886488438 +frame_id: "camera_color_optical_frame" +[INFO] [1733884887.313605]: received mask_obs with header seq: 3790 +stamp: + secs: 1733884887 + nsecs: 317583084 +frame_id: '' and grasped_object_mask header seq: 96516 +stamp: + secs: 1733884887 + nsecs: 88522434 +frame_id: "camera_color_optical_frame" +[INFO] [1733884887.509571]: received mask_obs with header seq: 3791 +stamp: + secs: 1733884887 + nsecs: 513100624 +frame_id: '' and grasped_object_mask header seq: 96522 +stamp: + secs: 1733884887 + nsecs: 290727377 +frame_id: "camera_color_optical_frame" +[INFO] [1733884887.703999]: received mask_obs with header seq: 3792 +stamp: + secs: 1733884887 + nsecs: 707786321 +frame_id: '' and grasped_object_mask header seq: 96527 +stamp: + secs: 1733884887 + nsecs: 459296942 +frame_id: "camera_color_optical_frame" +[INFO] [1733884887.901486]: received mask_obs with header seq: 3793 +stamp: + secs: 1733884887 + nsecs: 905427694 +frame_id: '' and grasped_object_mask header seq: 96533 +stamp: + secs: 1733884887 + nsecs: 661661386 +frame_id: "camera_color_optical_frame" +[INFO] [1733884888.117389]: received mask_obs with header seq: 3794 +stamp: + secs: 1733884888 + nsecs: 121491909 +frame_id: '' and grasped_object_mask header seq: 96539 +stamp: + secs: 1733884887 + nsecs: 864111662 +frame_id: "camera_color_optical_frame" +[INFO] [1733884888.312746]: received mask_obs with header seq: 3795 +stamp: + secs: 1733884888 + nsecs: 316858291 +frame_id: '' and grasped_object_mask header seq: 96546 +stamp: + secs: 1733884888 + nsecs: 97609282 +frame_id: "camera_color_optical_frame" +[INFO] [1733884888.521975]: received mask_obs with header seq: 3796 +stamp: + secs: 1733884888 + nsecs: 525854110 +frame_id: '' and grasped_object_mask header seq: 96551 +stamp: + secs: 1733884888 + nsecs: 263960123 +frame_id: "camera_color_optical_frame" +[INFO] [1733884888.569735]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733884888.728865]: received mask_obs with header seq: 3797 +stamp: + secs: 1733884888 + nsecs: 732359886 +frame_id: '' and grasped_object_mask header seq: 96558 +stamp: + secs: 1733884888 + nsecs: 496838331 +frame_id: "camera_color_optical_frame" +[INFO] [1733884889.357007]: Stored episode 1. +[INFO] [1733884889.357240]: Saving video... +[INFO] [1733884889.550521]: Video saved! +Starting episode 1 +[INFO] [1733884889.562724]: resetting environment +[INFO] [1733884889.566606]: cleared current plan +[INFO] [1733884889.567213]: moving to home +[INFO] [1733884892.370054]: reached home +[INFO] [1733884892.370548]: reset action history +[INFO] [1733884895.080823]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733884901.222222]: resetting environment +[INFO] [1733884901.227152]: cleared current plan +[INFO] [1733884901.227687]: moving to home +[INFO] [1733884902.229121]: reached home +[INFO] [1733884902.229522]: reset action history +[INFO] [1733884904.934596]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/episode_rosbags/episode_1_2024-12-10-21-41-44.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-608694.log +started roslaunch server http://158.130.50.37:45367/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [609137] +process[print_text-5]: started with pid [609138] +process[rosbag_record-6]: started with pid [609139] +started bagging! +[INFO] [1733884905.629975]: received mask_obs with header seq: 3798 +stamp: + secs: 1733884905 + nsecs: 633916378 +frame_id: '' and grasped_object_mask header seq: 97066 +stamp: + secs: 1733884905 + nsecs: 404913902 +frame_id: "camera_color_optical_frame" +[INFO] [1733884905.820637]: received mask_obs with header seq: 3799 +stamp: + secs: 1733884905 + nsecs: 822942972 +frame_id: '' and grasped_object_mask header seq: 97072 +stamp: + secs: 1733884905 + nsecs: 604750395 +frame_id: "camera_color_optical_frame" +[INFO] [1733884906.014913]: received mask_obs with header seq: 3800 +stamp: + secs: 1733884906 + nsecs: 18943071 +frame_id: '' and grasped_object_mask header seq: 97078 +stamp: + secs: 1733884905 + nsecs: 804586887 +frame_id: "camera_color_optical_frame" +[INFO] [1733884906.214207]: received mask_obs with header seq: 3801 +stamp: + secs: 1733884906 + nsecs: 218259811 +frame_id: '' and grasped_object_mask header seq: 97084 +stamp: + secs: 1733884906 + nsecs: 4423380 +frame_id: "camera_color_optical_frame" +[INFO] [1733884906.439085]: received mask_obs with header seq: 3802 +stamp: + secs: 1733884906 + nsecs: 443131446 +frame_id: '' and grasped_object_mask header seq: 97090 +stamp: + secs: 1733884906 + nsecs: 204258442 +frame_id: "camera_color_optical_frame" +[INFO] [1733884906.625689]: received mask_obs with header seq: 3803 +stamp: + secs: 1733884906 + nsecs: 629071712 +frame_id: '' and grasped_object_mask header seq: 97096 +stamp: + secs: 1733884906 + nsecs: 404093742 +frame_id: "camera_color_optical_frame" +[INFO] [1733884906.826294]: received mask_obs with header seq: 3804 +stamp: + secs: 1733884906 + nsecs: 830195188 +frame_id: '' and grasped_object_mask header seq: 97102 +stamp: + secs: 1733884906 + nsecs: 603928566 +frame_id: "camera_color_optical_frame" +[INFO] [1733884907.035490]: received mask_obs with header seq: 3805 +stamp: + secs: 1733884907 + nsecs: 39380073 +frame_id: '' and grasped_object_mask header seq: 97108 +stamp: + secs: 1733884906 + nsecs: 803764105 +frame_id: "camera_color_optical_frame" +[INFO] [1733884907.234102]: received mask_obs with header seq: 3806 +stamp: + secs: 1733884907 + nsecs: 238213062 +frame_id: '' and grasped_object_mask header seq: 97114 +stamp: + secs: 1733884907 + nsecs: 3598928 +frame_id: "camera_color_optical_frame" +[INFO] [1733884907.435515]: received mask_obs with header seq: 3807 +stamp: + secs: 1733884907 + nsecs: 439331293 +frame_id: '' and grasped_object_mask header seq: 97120 +stamp: + secs: 1733884907 + nsecs: 203434229 +frame_id: "camera_color_optical_frame" +[INFO] [1733884907.635381]: received mask_obs with header seq: 3808 +stamp: + secs: 1733884907 + nsecs: 639112472 +frame_id: '' and grasped_object_mask header seq: 97126 +stamp: + secs: 1733884907 + nsecs: 403269768 +frame_id: "camera_color_optical_frame" +[INFO] [1733884907.830862]: received mask_obs with header seq: 3809 +stamp: + secs: 1733884907 + nsecs: 835024356 +frame_id: '' and grasped_object_mask header seq: 97132 +stamp: + secs: 1733884907 + nsecs: 603105307 +frame_id: "camera_color_optical_frame" +[INFO] [1733884908.046886]: received mask_obs with header seq: 3810 +stamp: + secs: 1733884908 + nsecs: 50989389 +frame_id: '' and grasped_object_mask header seq: 97138 +stamp: + secs: 1733884907 + nsecs: 802940369 +frame_id: "camera_color_optical_frame" +[INFO] [1733884908.230598]: received mask_obs with header seq: 3811 +stamp: + secs: 1733884908 + nsecs: 234631776 +frame_id: '' and grasped_object_mask header seq: 97144 +stamp: + secs: 1733884908 + nsecs: 2777338 +frame_id: "camera_color_optical_frame" +[INFO] [1733884908.438238]: received mask_obs with header seq: 3812 +stamp: + secs: 1733884908 + nsecs: 442370653 +frame_id: '' and grasped_object_mask header seq: 97150 +stamp: + secs: 1733884908 + nsecs: 202615023 +frame_id: "camera_color_optical_frame" +[INFO] [1733884908.635590]: received mask_obs with header seq: 3813 +stamp: + secs: 1733884908 + nsecs: 639665126 +frame_id: '' and grasped_object_mask header seq: 97156 +stamp: + secs: 1733884908 + nsecs: 402452469 +frame_id: "camera_color_optical_frame" +[INFO] [1733884908.833457]: received mask_obs with header seq: 3814 +stamp: + secs: 1733884908 + nsecs: 837474107 +frame_id: '' and grasped_object_mask header seq: 97162 +stamp: + secs: 1733884908 + nsecs: 602290154 +frame_id: "camera_color_optical_frame" +[INFO] [1733884909.027877]: received mask_obs with header seq: 3815 +stamp: + secs: 1733884909 + nsecs: 31956195 +frame_id: '' and grasped_object_mask header seq: 97168 +stamp: + secs: 1733884908 + nsecs: 802127361 +frame_id: "camera_color_optical_frame" +[INFO] [1733884909.229324]: received mask_obs with header seq: 3816 +stamp: + secs: 1733884909 + nsecs: 233322143 +frame_id: '' and grasped_object_mask header seq: 97174 +stamp: + secs: 1733884909 + nsecs: 1965046 +frame_id: "camera_color_optical_frame" +[INFO] [1733884909.431735]: received mask_obs with header seq: 3817 +stamp: + secs: 1733884909 + nsecs: 435741424 +frame_id: '' and grasped_object_mask header seq: 97180 +stamp: + secs: 1733884909 + nsecs: 201804876 +frame_id: "camera_color_optical_frame" +[INFO] [1733884909.631200]: received mask_obs with header seq: 3818 +stamp: + secs: 1733884909 + nsecs: 635330915 +frame_id: '' and grasped_object_mask header seq: 97186 +stamp: + secs: 1733884909 + nsecs: 401643515 +frame_id: "camera_color_optical_frame" +[INFO] [1733884909.838854]: received mask_obs with header seq: 3819 +stamp: + secs: 1733884909 + nsecs: 842974662 +frame_id: '' and grasped_object_mask header seq: 97192 +stamp: + secs: 1733884909 + nsecs: 601482153 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 101136702 for nearest: 1733884909902457493 - target: 1733884909801320791 at idx 62 +[INFO] [1733884910.056675]: received mask_obs with header seq: 3820 +stamp: + secs: 1733884910 + nsecs: 59767961 +frame_id: '' and grasped_object_mask header seq: 97198 +stamp: + secs: 1733884909 + nsecs: 801320791 +frame_id: "camera_color_optical_frame" +[INFO] [1733884910.265752]: received mask_obs with header seq: 3821 +stamp: + secs: 1733884910 + nsecs: 269490242 +frame_id: '' and grasped_object_mask header seq: 97205 +stamp: + secs: 1733884910 + nsecs: 34466505 +frame_id: "camera_color_optical_frame" +[INFO] [1733884910.455480]: received mask_obs with header seq: 3822 +stamp: + secs: 1733884910 + nsecs: 458603382 +frame_id: '' and grasped_object_mask header seq: 97211 +stamp: + secs: 1733884910 + nsecs: 234305859 +frame_id: "camera_color_optical_frame" +[INFO] [1733884910.656389]: received mask_obs with header seq: 3823 +stamp: + secs: 1733884910 + nsecs: 659624814 +frame_id: '' and grasped_object_mask header seq: 97217 +stamp: + secs: 1733884910 + nsecs: 434144974 +frame_id: "camera_color_optical_frame" +[INFO] [1733884910.859023]: received mask_obs with header seq: 3824 +stamp: + secs: 1733884910 + nsecs: 863105773 +frame_id: '' and grasped_object_mask header seq: 97222 +stamp: + secs: 1733884910 + nsecs: 600678444 +frame_id: "camera_color_optical_frame" +[INFO] [1733884911.059437]: received mask_obs with header seq: 3825 +stamp: + secs: 1733884911 + nsecs: 63371419 +frame_id: '' and grasped_object_mask header seq: 97228 +stamp: + secs: 1733884910 + nsecs: 800517559 +frame_id: "camera_color_optical_frame" +[INFO] [1733884911.263605]: received mask_obs with header seq: 3826 +stamp: + secs: 1733884911 + nsecs: 267624378 +frame_id: '' and grasped_object_mask header seq: 97235 +stamp: + secs: 1733884911 + nsecs: 33661366 +frame_id: "camera_color_optical_frame" +[INFO] [1733884911.469067]: received mask_obs with header seq: 3827 +stamp: + secs: 1733884911 + nsecs: 473039627 +frame_id: '' and grasped_object_mask header seq: 97241 +stamp: + secs: 1733884911 + nsecs: 233494282 +frame_id: "camera_color_optical_frame" +[INFO] [1733884911.673348]: received mask_obs with header seq: 3828 +stamp: + secs: 1733884911 + nsecs: 677450180 +frame_id: '' and grasped_object_mask header seq: 97247 +stamp: + secs: 1733884911 + nsecs: 433326960 +frame_id: "camera_color_optical_frame" +[INFO] [1733884911.863875]: received mask_obs with header seq: 3829 +stamp: + secs: 1733884911 + nsecs: 867802619 +frame_id: '' and grasped_object_mask header seq: 97253 +stamp: + secs: 1733884911 + nsecs: 633158922 +frame_id: "camera_color_optical_frame" +[INFO] [1733884912.053623]: received mask_obs with header seq: 3830 +stamp: + secs: 1733884912 + nsecs: 57764291 +frame_id: '' and grasped_object_mask header seq: 97259 +stamp: + secs: 1733884911 + nsecs: 832990885 +frame_id: "camera_color_optical_frame" +[INFO] [1733884912.232070]: received mask_obs with header seq: 3831 +stamp: + secs: 1733884912 + nsecs: 236161470 +frame_id: '' and grasped_object_mask header seq: 97264 +stamp: + secs: 1733884911 + nsecs: 999520302 +frame_id: "camera_color_optical_frame" +[INFO] [1733884912.430474]: received mask_obs with header seq: 3832 +stamp: + secs: 1733884912 + nsecs: 434447288 +frame_id: '' and grasped_object_mask header seq: 97270 +stamp: + secs: 1733884912 + nsecs: 199361801 +frame_id: "camera_color_optical_frame" +[INFO] [1733884912.639371]: received mask_obs with header seq: 3833 +stamp: + secs: 1733884912 + nsecs: 643260002 +frame_id: '' and grasped_object_mask header seq: 97276 +stamp: + secs: 1733884912 + nsecs: 399203777 +frame_id: "camera_color_optical_frame" +[INFO] [1733884912.831482]: received mask_obs with header seq: 3834 +stamp: + secs: 1733884912 + nsecs: 835464239 +frame_id: '' and grasped_object_mask header seq: 97282 +stamp: + secs: 1733884912 + nsecs: 599045992 +frame_id: "camera_color_optical_frame" +[INFO] [1733884913.039563]: received mask_obs with header seq: 3835 +stamp: + secs: 1733884913 + nsecs: 43385505 +frame_id: '' and grasped_object_mask header seq: 97288 +stamp: + secs: 1733884912 + nsecs: 798888445 +frame_id: "camera_color_optical_frame" +[INFO] [1733884913.230033]: received mask_obs with header seq: 3836 +stamp: + secs: 1733884913 + nsecs: 234165906 +frame_id: '' and grasped_object_mask header seq: 97294 +stamp: + secs: 1733884912 + nsecs: 998731613 +frame_id: "camera_color_optical_frame" +[INFO] [1733884913.455879]: received mask_obs with header seq: 3837 +stamp: + secs: 1733884913 + nsecs: 459656238 +frame_id: '' and grasped_object_mask header seq: 97300 +stamp: + secs: 1733884913 + nsecs: 198571444 +frame_id: "camera_color_optical_frame" +[INFO] [1733884913.640673]: received mask_obs with header seq: 3838 +stamp: + secs: 1733884913 + nsecs: 644545793 +frame_id: '' and grasped_object_mask header seq: 97307 +stamp: + secs: 1733884913 + nsecs: 431719065 +frame_id: "camera_color_optical_frame" +[INFO] [1733884913.836301]: received mask_obs with header seq: 3839 +stamp: + secs: 1733884913 + nsecs: 840420007 +frame_id: '' and grasped_object_mask header seq: 97312 +stamp: + secs: 1733884913 + nsecs: 598252058 +frame_id: "camera_color_optical_frame" +[INFO] [1733884914.033010]: received mask_obs with header seq: 3840 +stamp: + secs: 1733884914 + nsecs: 36917209 +frame_id: '' and grasped_object_mask header seq: 97318 +stamp: + secs: 1733884913 + nsecs: 798092365 +frame_id: "camera_color_optical_frame" +[INFO] [1733884914.234450]: received mask_obs with header seq: 3841 +stamp: + secs: 1733884914 + nsecs: 238603353 +frame_id: '' and grasped_object_mask header seq: 97324 +stamp: + secs: 1733884913 + nsecs: 997933626 +frame_id: "camera_color_optical_frame" +[INFO] [1733884914.431275]: received mask_obs with header seq: 3842 +stamp: + secs: 1733884914 + nsecs: 435426473 +frame_id: '' and grasped_object_mask header seq: 97330 +stamp: + secs: 1733884914 + nsecs: 197781086 +frame_id: "camera_color_optical_frame" +[INFO] [1733884914.643146]: received mask_obs with header seq: 3843 +stamp: + secs: 1733884914 + nsecs: 647207975 +frame_id: '' and grasped_object_mask header seq: 97336 +stamp: + secs: 1733884914 + nsecs: 397627592 +frame_id: "camera_color_optical_frame" +[INFO] [1733884914.834840]: received mask_obs with header seq: 3844 +stamp: + secs: 1733884914 + nsecs: 838922739 +frame_id: '' and grasped_object_mask header seq: 97342 +stamp: + secs: 1733884914 + nsecs: 597474813 +frame_id: "camera_color_optical_frame" +[INFO] [1733884915.036972]: received mask_obs with header seq: 3845 +stamp: + secs: 1733884915 + nsecs: 40968179 +frame_id: '' and grasped_object_mask header seq: 97348 +stamp: + secs: 1733884914 + nsecs: 797322512 +frame_id: "camera_color_optical_frame" +[INFO] [1733884915.236012]: received mask_obs with header seq: 3846 +stamp: + secs: 1733884915 + nsecs: 239950656 +frame_id: '' and grasped_object_mask header seq: 97354 +stamp: + secs: 1733884914 + nsecs: 997169256 +frame_id: "camera_color_optical_frame" +[INFO] [1733884915.437162]: received mask_obs with header seq: 3847 +stamp: + secs: 1733884915 + nsecs: 441229343 +frame_id: '' and grasped_object_mask header seq: 97360 +stamp: + secs: 1733884915 + nsecs: 197010279 +frame_id: "camera_color_optical_frame" +[INFO] [1733884915.635120]: received mask_obs with header seq: 3848 +stamp: + secs: 1733884915 + nsecs: 639147520 +frame_id: '' and grasped_object_mask header seq: 97366 +stamp: + secs: 1733884915 + nsecs: 396852255 +frame_id: "camera_color_optical_frame" +[INFO] [1733884915.840319]: received mask_obs with header seq: 3849 +stamp: + secs: 1733884915 + nsecs: 844407796 +frame_id: '' and grasped_object_mask header seq: 97372 +stamp: + secs: 1733884915 + nsecs: 596693277 +frame_id: "camera_color_optical_frame" +[INFO] [1733884916.046007]: received mask_obs with header seq: 3850 +stamp: + secs: 1733884916 + nsecs: 49173355 +frame_id: '' and grasped_object_mask header seq: 97378 +stamp: + secs: 1733884915 + nsecs: 796534777 +frame_id: "camera_color_optical_frame" +[INFO] [1733884916.260567]: received mask_obs with header seq: 3851 +stamp: + secs: 1733884916 + nsecs: 264444112 +frame_id: '' and grasped_object_mask header seq: 97385 +stamp: + secs: 1733884916 + nsecs: 29680729 +frame_id: "camera_color_optical_frame" +[INFO] [1733884916.468112]: received mask_obs with header seq: 3852 +stamp: + secs: 1733884916 + nsecs: 472095251 +frame_id: '' and grasped_object_mask header seq: 97391 +stamp: + secs: 1733884916 + nsecs: 229513884 +frame_id: "camera_color_optical_frame" +[INFO] [1733884916.661172]: received mask_obs with header seq: 3853 +stamp: + secs: 1733884916 + nsecs: 665211915 +frame_id: '' and grasped_object_mask header seq: 97397 +stamp: + secs: 1733884916 + nsecs: 429347277 +frame_id: "camera_color_optical_frame" +[INFO] [1733884916.857342]: received mask_obs with header seq: 3854 +stamp: + secs: 1733884916 + nsecs: 861475229 +frame_id: '' and grasped_object_mask header seq: 97403 +stamp: + secs: 1733884916 + nsecs: 629179955 +frame_id: "camera_color_optical_frame" +[INFO] [1733884917.052969]: received mask_obs with header seq: 3855 +stamp: + secs: 1733884917 + nsecs: 57024002 +frame_id: '' and grasped_object_mask header seq: 97409 +stamp: + secs: 1733884916 + nsecs: 829012632 +frame_id: "camera_color_optical_frame" +[INFO] [1733884917.257282]: received mask_obs with header seq: 3856 +stamp: + secs: 1733884917 + nsecs: 261399984 +frame_id: '' and grasped_object_mask header seq: 97415 +stamp: + secs: 1733884917 + nsecs: 28847456 +frame_id: "camera_color_optical_frame" +[INFO] [1733884917.451139]: received mask_obs with header seq: 3857 +stamp: + secs: 1733884917 + nsecs: 455263376 +frame_id: '' and grasped_object_mask header seq: 97421 +stamp: + secs: 1733884917 + nsecs: 228687525 +frame_id: "camera_color_optical_frame" +[INFO] [1733884917.649488]: received mask_obs with header seq: 3858 +stamp: + secs: 1733884917 + nsecs: 653591632 +frame_id: '' and grasped_object_mask header seq: 97426 +stamp: + secs: 1733884917 + nsecs: 395219803 +frame_id: "camera_color_optical_frame" +[INFO] [1733884917.828954]: received mask_obs with header seq: 3859 +stamp: + secs: 1733884917 + nsecs: 832791328 +frame_id: '' and grasped_object_mask header seq: 97432 +stamp: + secs: 1733884917 + nsecs: 595058918 +frame_id: "camera_color_optical_frame" +[INFO] [1733884918.059933]: received mask_obs with header seq: 3860 +stamp: + secs: 1733884918 + nsecs: 63737630 +frame_id: '' and grasped_object_mask header seq: 97438 +stamp: + secs: 1733884917 + nsecs: 794899225 +frame_id: "camera_color_optical_frame" +[INFO] [1733884918.253516]: received mask_obs with header seq: 3861 +stamp: + secs: 1733884918 + nsecs: 257407903 +frame_id: '' and grasped_object_mask header seq: 97445 +stamp: + secs: 1733884918 + nsecs: 28042793 +frame_id: "camera_color_optical_frame" +[INFO] [1733884918.436131]: received mask_obs with header seq: 3862 +stamp: + secs: 1733884918 + nsecs: 440190315 +frame_id: '' and grasped_object_mask header seq: 97450 +stamp: + secs: 1733884918 + nsecs: 194571733 +frame_id: "camera_color_optical_frame" +[INFO] [1733884918.629805]: received mask_obs with header seq: 3863 +stamp: + secs: 1733884918 + nsecs: 633569955 +frame_id: '' and grasped_object_mask header seq: 97456 +stamp: + secs: 1733884918 + nsecs: 394405842 +frame_id: "camera_color_optical_frame" +[INFO] [1733884918.829959]: received mask_obs with header seq: 3864 +stamp: + secs: 1733884918 + nsecs: 834081649 +frame_id: '' and grasped_object_mask header seq: 97462 +stamp: + secs: 1733884918 + nsecs: 594239712 +frame_id: "camera_color_optical_frame" +[INFO] [1733884919.039090]: received mask_obs with header seq: 3865 +stamp: + secs: 1733884919 + nsecs: 42996406 +frame_id: '' and grasped_object_mask header seq: 97468 +stamp: + secs: 1733884918 + nsecs: 794073582 +frame_id: "camera_color_optical_frame" +[INFO] [1733884919.238369]: received mask_obs with header seq: 3866 +stamp: + secs: 1733884919 + nsecs: 242186307 +frame_id: '' and grasped_object_mask header seq: 97474 +stamp: + secs: 1733884918 + nsecs: 993907928 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 108570832 for nearest: 1733884919302310961 - target: 1733884919193740129 at idx 198 +[INFO] [1733884919.447690]: received mask_obs with header seq: 3867 +stamp: + secs: 1733884919 + nsecs: 451685190 +frame_id: '' and grasped_object_mask header seq: 97480 +stamp: + secs: 1733884919 + nsecs: 193740129 +frame_id: "camera_color_optical_frame" +[INFO] [1733884919.630141]: received mask_obs with header seq: 3868 +stamp: + secs: 1733884919 + nsecs: 633908987 +frame_id: '' and grasped_object_mask header seq: 97486 +stamp: + secs: 1733884919 + nsecs: 393572569 +frame_id: "camera_color_optical_frame" +[INFO] [1733884919.848814]: received mask_obs with header seq: 3869 +stamp: + secs: 1733884919 + nsecs: 852764606 +frame_id: '' and grasped_object_mask header seq: 97492 +stamp: + secs: 1733884919 + nsecs: 593404055 +frame_id: "camera_color_optical_frame" +[INFO] [1733884920.043409]: received mask_obs with header seq: 3870 +stamp: + secs: 1733884920 + nsecs: 45319318 +frame_id: '' and grasped_object_mask header seq: 97498 +stamp: + secs: 1733884919 + nsecs: 793235540 +frame_id: "camera_color_optical_frame" +[INFO] [1733884920.233330]: received mask_obs with header seq: 3871 +stamp: + secs: 1733884920 + nsecs: 237191200 +frame_id: '' and grasped_object_mask header seq: 97505 +stamp: + secs: 1733884920 + nsecs: 26376963 +frame_id: "camera_color_optical_frame" +[INFO] [1733884920.436797]: received mask_obs with header seq: 3872 +stamp: + secs: 1733884920 + nsecs: 440643548 +frame_id: '' and grasped_object_mask header seq: 97510 +stamp: + secs: 1733884920 + nsecs: 192914248 +frame_id: "camera_color_optical_frame" +[INFO] [1733884920.633637]: received mask_obs with header seq: 3873 +stamp: + secs: 1733884920 + nsecs: 637761354 +frame_id: '' and grasped_object_mask header seq: 97516 +stamp: + secs: 1733884920 + nsecs: 392760992 +frame_id: "camera_color_optical_frame" +[INFO] [1733884920.835433]: received mask_obs with header seq: 3874 +stamp: + secs: 1733884920 + nsecs: 839205026 +frame_id: '' and grasped_object_mask header seq: 97522 +stamp: + secs: 1733884920 + nsecs: 592607260 +frame_id: "camera_color_optical_frame" +[INFO] [1733884921.040017]: received mask_obs with header seq: 3875 +stamp: + secs: 1733884921 + nsecs: 43884515 +frame_id: '' and grasped_object_mask header seq: 97529 +stamp: + secs: 1733884920 + nsecs: 825761795 +frame_id: "camera_color_optical_frame" +[INFO] [1733884921.238598]: received mask_obs with header seq: 3876 +stamp: + secs: 1733884921 + nsecs: 242564678 +frame_id: '' and grasped_object_mask header seq: 97534 +stamp: + secs: 1733884920 + nsecs: 992299080 +frame_id: "camera_color_optical_frame" +[INFO] [1733884921.440842]: received mask_obs with header seq: 3877 +stamp: + secs: 1733884921 + nsecs: 443778276 +frame_id: '' and grasped_object_mask header seq: 97540 +stamp: + secs: 1733884921 + nsecs: 192138910 +frame_id: "camera_color_optical_frame" +[INFO] [1733884921.641742]: received mask_obs with header seq: 3878 +stamp: + secs: 1733884921 + nsecs: 644947528 +frame_id: '' and grasped_object_mask header seq: 97547 +stamp: + secs: 1733884921 + nsecs: 425285816 +frame_id: "camera_color_optical_frame" +[INFO] [1733884921.848282]: received mask_obs with header seq: 3879 +stamp: + secs: 1733884921 + nsecs: 852399110 +frame_id: '' and grasped_object_mask header seq: 97553 +stamp: + secs: 1733884921 + nsecs: 625125647 +frame_id: "camera_color_optical_frame" +[INFO] [1733884922.045616]: received mask_obs with header seq: 3880 +stamp: + secs: 1733884922 + nsecs: 49598693 +frame_id: '' and grasped_object_mask header seq: 97559 +stamp: + secs: 1733884921 + nsecs: 824965715 +frame_id: "camera_color_optical_frame" +[INFO] [1733884922.239854]: received mask_obs with header seq: 3881 +stamp: + secs: 1733884922 + nsecs: 242361307 +frame_id: '' and grasped_object_mask header seq: 97565 +stamp: + secs: 1733884922 + nsecs: 24806261 +frame_id: "camera_color_optical_frame" +[INFO] [1733884922.441087]: received mask_obs with header seq: 3882 +stamp: + secs: 1733884922 + nsecs: 444544553 +frame_id: '' and grasped_object_mask header seq: 97571 +stamp: + secs: 1733884922 + nsecs: 224648952 +frame_id: "camera_color_optical_frame" +[INFO] [1733884922.645194]: received mask_obs with header seq: 3883 +stamp: + secs: 1733884922 + nsecs: 648879051 +frame_id: '' and grasped_object_mask header seq: 97577 +stamp: + secs: 1733884922 + nsecs: 424490452 +frame_id: "camera_color_optical_frame" +[INFO] [1733884922.841040]: received mask_obs with header seq: 3884 +stamp: + secs: 1733884922 + nsecs: 844765424 +frame_id: '' and grasped_object_mask header seq: 97583 +stamp: + secs: 1733884922 + nsecs: 624332428 +frame_id: "camera_color_optical_frame" +[INFO] [1733884923.055542]: received mask_obs with header seq: 3885 +stamp: + secs: 1733884923 + nsecs: 59583663 +frame_id: '' and grasped_object_mask header seq: 97589 +stamp: + secs: 1733884922 + nsecs: 824174643 +frame_id: "camera_color_optical_frame" +[INFO] [1733884923.245698]: received mask_obs with header seq: 3886 +stamp: + secs: 1733884923 + nsecs: 249515295 +frame_id: '' and grasped_object_mask header seq: 97595 +stamp: + secs: 1733884923 + nsecs: 24015665 +frame_id: "camera_color_optical_frame" +[INFO] [1733884923.450772]: received mask_obs with header seq: 3887 +stamp: + secs: 1733884923 + nsecs: 454915761 +frame_id: '' and grasped_object_mask header seq: 97601 +stamp: + secs: 1733884923 + nsecs: 223854542 +frame_id: "camera_color_optical_frame" +[INFO] [1733884923.636211]: received mask_obs with header seq: 3888 +stamp: + secs: 1733884923 + nsecs: 640385389 +frame_id: '' and grasped_object_mask header seq: 97607 +stamp: + secs: 1733884923 + nsecs: 423694372 +frame_id: "camera_color_optical_frame" +[INFO] [1733884923.848027]: received mask_obs with header seq: 3889 +stamp: + secs: 1733884923 + nsecs: 852103471 +frame_id: '' and grasped_object_mask header seq: 97613 +stamp: + secs: 1733884923 + nsecs: 623533249 +frame_id: "camera_color_optical_frame" +[INFO] [1733884924.047913]: received mask_obs with header seq: 3890 +stamp: + secs: 1733884924 + nsecs: 51492929 +frame_id: '' and grasped_object_mask header seq: 97619 +stamp: + secs: 1733884923 + nsecs: 823371887 +frame_id: "camera_color_optical_frame" +[INFO] [1733884924.241860]: received mask_obs with header seq: 3891 +stamp: + secs: 1733884924 + nsecs: 245893955 +frame_id: '' and grasped_object_mask header seq: 97625 +stamp: + secs: 1733884924 + nsecs: 23208380 +frame_id: "camera_color_optical_frame" +[INFO] [1733884924.439124]: received mask_obs with header seq: 3892 +stamp: + secs: 1733884924 + nsecs: 443058252 +frame_id: '' and grasped_object_mask header seq: 97631 +stamp: + secs: 1733884924 + nsecs: 223040342 +frame_id: "camera_color_optical_frame" +[INFO] [1733884924.638304]: received mask_obs with header seq: 3893 +stamp: + secs: 1733884924 + nsecs: 640487909 +frame_id: '' and grasped_object_mask header seq: 97637 +stamp: + secs: 1733884924 + nsecs: 422871828 +frame_id: "camera_color_optical_frame" +[INFO] [1733884924.845133]: received mask_obs with header seq: 3894 +stamp: + secs: 1733884924 + nsecs: 848813533 +frame_id: '' and grasped_object_mask header seq: 97643 +stamp: + secs: 1733884924 + nsecs: 622704268 +frame_id: "camera_color_optical_frame" +[INFO] [1733884925.047449]: received mask_obs with header seq: 3895 +stamp: + secs: 1733884925 + nsecs: 50188779 +frame_id: '' and grasped_object_mask header seq: 97649 +stamp: + secs: 1733884924 + nsecs: 822535515 +frame_id: "camera_color_optical_frame" +[INFO] [1733884925.250065]: received mask_obs with header seq: 3896 +stamp: + secs: 1733884925 + nsecs: 253897666 +frame_id: '' and grasped_object_mask header seq: 97655 +stamp: + secs: 1733884925 + nsecs: 22369385 +frame_id: "camera_color_optical_frame" +[INFO] [1733884925.448022]: received mask_obs with header seq: 3897 +stamp: + secs: 1733884925 + nsecs: 451314449 +frame_id: '' and grasped_object_mask header seq: 97661 +stamp: + secs: 1733884925 + nsecs: 222209215 +frame_id: "camera_color_optical_frame" +[INFO] [1733884925.648725]: received mask_obs with header seq: 3898 +stamp: + secs: 1733884925 + nsecs: 652821302 +frame_id: '' and grasped_object_mask header seq: 97667 +stamp: + secs: 1733884925 + nsecs: 422049046 +frame_id: "camera_color_optical_frame" +[INFO] [1733884925.843763]: received mask_obs with header seq: 3899 +stamp: + secs: 1733884925 + nsecs: 847902774 +frame_id: '' and grasped_object_mask header seq: 97673 +stamp: + secs: 1733884925 + nsecs: 621889353 +frame_id: "camera_color_optical_frame" +[INFO] [1733884926.043173]: received mask_obs with header seq: 3900 +stamp: + secs: 1733884926 + nsecs: 46866178 +frame_id: '' and grasped_object_mask header seq: 97679 +stamp: + secs: 1733884925 + nsecs: 821730375 +frame_id: "camera_color_optical_frame" +[INFO] [1733884926.242093]: received mask_obs with header seq: 3901 +stamp: + secs: 1733884926 + nsecs: 246237754 +frame_id: '' and grasped_object_mask header seq: 97685 +stamp: + secs: 1733884926 + nsecs: 21570206 +frame_id: "camera_color_optical_frame" +[INFO] [1733884926.439326]: received mask_obs with header seq: 3902 +stamp: + secs: 1733884926 + nsecs: 443451404 +frame_id: '' and grasped_object_mask header seq: 97691 +stamp: + secs: 1733884926 + nsecs: 221409798 +frame_id: "camera_color_optical_frame" +[INFO] [1733884926.636677]: received mask_obs with header seq: 3903 +stamp: + secs: 1733884926 + nsecs: 640371799 +frame_id: '' and grasped_object_mask header seq: 97697 +stamp: + secs: 1733884926 + nsecs: 421249151 +frame_id: "camera_color_optical_frame" +[INFO] [1733884926.836905]: received mask_obs with header seq: 3904 +stamp: + secs: 1733884926 + nsecs: 840833425 +frame_id: '' and grasped_object_mask header seq: 97703 +stamp: + secs: 1733884926 + nsecs: 621088982 +frame_id: "camera_color_optical_frame" +[INFO] [1733884927.040583]: received mask_obs with header seq: 3905 +stamp: + secs: 1733884927 + nsecs: 44518709 +frame_id: '' and grasped_object_mask header seq: 97709 +stamp: + secs: 1733884926 + nsecs: 820928574 +frame_id: "camera_color_optical_frame" +[INFO] [1733884927.250040]: received mask_obs with header seq: 3906 +stamp: + secs: 1733884927 + nsecs: 254226684 +frame_id: '' and grasped_object_mask header seq: 97715 +stamp: + secs: 1733884927 + nsecs: 20769119 +frame_id: "camera_color_optical_frame" +[INFO] [1733884927.448643]: received mask_obs with header seq: 3907 +stamp: + secs: 1733884927 + nsecs: 452789545 +frame_id: '' and grasped_object_mask header seq: 97721 +stamp: + secs: 1733884927 + nsecs: 220606804 +frame_id: "camera_color_optical_frame" +[INFO] [1733884927.642720]: received mask_obs with header seq: 3908 +stamp: + secs: 1733884927 + nsecs: 646874904 +frame_id: '' and grasped_object_mask header seq: 97727 +stamp: + secs: 1733884927 + nsecs: 420444489 +frame_id: "camera_color_optical_frame" +[INFO] [1733884927.837024]: received mask_obs with header seq: 3909 +stamp: + secs: 1733884927 + nsecs: 841102838 +frame_id: '' and grasped_object_mask header seq: 97733 +stamp: + secs: 1733884927 + nsecs: 620282173 +frame_id: "camera_color_optical_frame" +[INFO] [1733884928.037079]: received mask_obs with header seq: 3910 +stamp: + secs: 1733884928 + nsecs: 39604902 +frame_id: '' and grasped_object_mask header seq: 97739 +stamp: + secs: 1733884927 + nsecs: 820119619 +frame_id: "camera_color_optical_frame" +[INFO] [1733884928.235466]: received mask_obs with header seq: 3911 +stamp: + secs: 1733884928 + nsecs: 239503383 +frame_id: '' and grasped_object_mask header seq: 97745 +stamp: + secs: 1733884928 + nsecs: 19957304 +frame_id: "camera_color_optical_frame" +[INFO] [1733884928.437112]: received mask_obs with header seq: 3912 +stamp: + secs: 1733884928 + nsecs: 440185070 +frame_id: '' and grasped_object_mask header seq: 97751 +stamp: + secs: 1733884928 + nsecs: 219795227 +frame_id: "camera_color_optical_frame" +[INFO] [1733884928.644240]: received mask_obs with header seq: 3913 +stamp: + secs: 1733884928 + nsecs: 646902084 +frame_id: '' and grasped_object_mask header seq: 97757 +stamp: + secs: 1733884928 + nsecs: 419633865 +frame_id: "camera_color_optical_frame" +[INFO] [1733884928.843547]: received mask_obs with header seq: 3914 +stamp: + secs: 1733884928 + nsecs: 846358776 +frame_id: '' and grasped_object_mask header seq: 97763 +stamp: + secs: 1733884928 + nsecs: 619471788 +frame_id: "camera_color_optical_frame" +[INFO] [1733884929.034195]: received mask_obs with header seq: 3915 +stamp: + secs: 1733884929 + nsecs: 38292884 +frame_id: '' and grasped_object_mask header seq: 97769 +stamp: + secs: 1733884928 + nsecs: 819309235 +frame_id: "camera_color_optical_frame" +[INFO] [1733884929.235779]: received mask_obs with header seq: 3916 +stamp: + secs: 1733884929 + nsecs: 238748073 +frame_id: '' and grasped_object_mask header seq: 97775 +stamp: + secs: 1733884929 + nsecs: 19146919 +frame_id: "camera_color_optical_frame" +[INFO] [1733884929.437418]: received mask_obs with header seq: 3917 +stamp: + secs: 1733884929 + nsecs: 441382884 +frame_id: '' and grasped_object_mask header seq: 97781 +stamp: + secs: 1733884929 + nsecs: 218985081 +frame_id: "camera_color_optical_frame" +[INFO] [1733884929.634641]: received mask_obs with header seq: 3918 +stamp: + secs: 1733884929 + nsecs: 638832330 +frame_id: '' and grasped_object_mask header seq: 97787 +stamp: + secs: 1733884929 + nsecs: 418823242 +frame_id: "camera_color_optical_frame" +[INFO] [1733884929.836145]: received mask_obs with header seq: 3919 +stamp: + secs: 1733884929 + nsecs: 839871168 +frame_id: '' and grasped_object_mask header seq: 97793 +stamp: + secs: 1733884929 + nsecs: 618662357 +frame_id: "camera_color_optical_frame" +[INFO] [1733884930.039756]: received mask_obs with header seq: 3920 +stamp: + secs: 1733884930 + nsecs: 43891668 +frame_id: '' and grasped_object_mask header seq: 97799 +stamp: + secs: 1733884929 + nsecs: 818500519 +frame_id: "camera_color_optical_frame" +[INFO] [1733884930.237904]: received mask_obs with header seq: 3921 +stamp: + secs: 1733884930 + nsecs: 242030620 +frame_id: '' and grasped_object_mask header seq: 97805 +stamp: + secs: 1733884930 + nsecs: 18339396 +frame_id: "camera_color_optical_frame" +[INFO] [1733884930.439235]: received mask_obs with header seq: 3922 +stamp: + secs: 1733884930 + nsecs: 443386793 +frame_id: '' and grasped_object_mask header seq: 97811 +stamp: + secs: 1733884930 + nsecs: 218181610 +frame_id: "camera_color_optical_frame" +[INFO] [1733884930.632968]: received mask_obs with header seq: 3923 +stamp: + secs: 1733884930 + nsecs: 636525869 +frame_id: '' and grasped_object_mask header seq: 97817 +stamp: + secs: 1733884930 + nsecs: 418024302 +frame_id: "camera_color_optical_frame" +[INFO] [1733884930.833628]: received mask_obs with header seq: 3924 +stamp: + secs: 1733884930 + nsecs: 837369918 +frame_id: '' and grasped_object_mask header seq: 97823 +stamp: + secs: 1733884930 + nsecs: 617867231 +frame_id: "camera_color_optical_frame" +[INFO] [1733884931.039932]: received mask_obs with header seq: 3925 +stamp: + secs: 1733884931 + nsecs: 43570756 +frame_id: '' and grasped_object_mask header seq: 97829 +stamp: + secs: 1733884930 + nsecs: 817711353 +frame_id: "camera_color_optical_frame" +[INFO] [1733884931.245151]: received mask_obs with header seq: 3926 +stamp: + secs: 1733884931 + nsecs: 249249696 +frame_id: '' and grasped_object_mask header seq: 97835 +stamp: + secs: 1733884931 + nsecs: 17554045 +frame_id: "camera_color_optical_frame" +[INFO] [1733884931.439255]: received mask_obs with header seq: 3927 +stamp: + secs: 1733884931 + nsecs: 443403244 +frame_id: '' and grasped_object_mask header seq: 97841 +stamp: + secs: 1733884931 + nsecs: 217392921 +frame_id: "camera_color_optical_frame" +[INFO] [1733884931.638921]: received mask_obs with header seq: 3928 +stamp: + secs: 1733884931 + nsecs: 642913579 +frame_id: '' and grasped_object_mask header seq: 97847 +stamp: + secs: 1733884931 + nsecs: 417232037 +frame_id: "camera_color_optical_frame" +[INFO] [1733884931.841230]: received mask_obs with header seq: 3929 +stamp: + secs: 1733884931 + nsecs: 845314264 +frame_id: '' and grasped_object_mask header seq: 97853 +stamp: + secs: 1733884931 + nsecs: 617071390 +frame_id: "camera_color_optical_frame" +[INFO] [1733884932.039738]: received mask_obs with header seq: 3930 +stamp: + secs: 1733884932 + nsecs: 43606519 +frame_id: '' and grasped_object_mask header seq: 97859 +stamp: + secs: 1733884931 + nsecs: 816910744 +frame_id: "camera_color_optical_frame" +[INFO] [1733884932.237676]: received mask_obs with header seq: 3931 +stamp: + secs: 1733884932 + nsecs: 241330385 +frame_id: '' and grasped_object_mask header seq: 97865 +stamp: + secs: 1733884932 + nsecs: 16749620 +frame_id: "camera_color_optical_frame" +[INFO] [1733884932.435418]: received mask_obs with header seq: 3932 +stamp: + secs: 1733884932 + nsecs: 439218997 +frame_id: '' and grasped_object_mask header seq: 97871 +stamp: + secs: 1733884932 + nsecs: 216580391 +frame_id: "camera_color_optical_frame" +[INFO] [1733884932.488089]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1733884932.635031]: received mask_obs with header seq: 3933 +stamp: + secs: 1733884932 + nsecs: 636618137 +frame_id: '' and grasped_object_mask header seq: 97877 +stamp: + secs: 1733884932 + nsecs: 416410923 +frame_id: "camera_color_optical_frame" +[INFO] [1733884933.257049]: Stored episode 2. +[INFO] [1733884933.257700]: Saving video... +[INFO] [1733884933.833060]: Video saved! +Starting episode 2 +[INFO] [1733884933.860649]: resetting environment +[INFO] [1733884933.874594]: cleared current plan +[INFO] [1733884933.874855]: moving to home +[WARN] [1733884933.824071]: Plan exhausted +[WARN] [1733884933.865243]: Plan exhausted +[INFO] [1733884936.879929]: reached home +[INFO] [1733884936.883723]: reset action history +[INFO] [1733884939.608459]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733884967.765305]: resetting environment +[INFO] [1733884967.779154]: cleared current plan +[INFO] [1733884967.779712]: moving to home +[INFO] [1733884968.781403]: reached home +[INFO] [1733884968.782215]: reset action history +[INFO] [1733884971.489443]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/episode_rosbags/episode_2_2024-12-10-21-42-51.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-608694.log +started roslaunch server http://158.130.50.37:36007/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [609299] +process[print_text-8]: started with pid [609300] +process[rosbag_record-9]: started with pid [609324] +started bagging! +[INFO] [1733884972.181809]: received mask_obs with header seq: 3934 +stamp: + secs: 1733884972 + nsecs: 185477018 +frame_id: '' and grasped_object_mask header seq: 99065 +stamp: + secs: 1733884971 + nsecs: 984533072 +frame_id: "camera_color_optical_frame" +[INFO] [1733884972.391941]: received mask_obs with header seq: 3935 +stamp: + secs: 1733884972 + nsecs: 396171569 +frame_id: '' and grasped_object_mask header seq: 99071 +stamp: + secs: 1733884972 + nsecs: 184366941 +frame_id: "camera_color_optical_frame" +[INFO] [1733884972.586051]: received mask_obs with header seq: 3936 +stamp: + secs: 1733884972 + nsecs: 589885234 +frame_id: '' and grasped_object_mask header seq: 99077 +stamp: + secs: 1733884972 + nsecs: 384199858 +frame_id: "camera_color_optical_frame" +[INFO] [1733884972.790778]: received mask_obs with header seq: 3937 +stamp: + secs: 1733884972 + nsecs: 794766426 +frame_id: '' and grasped_object_mask header seq: 99083 +stamp: + secs: 1733884972 + nsecs: 584032774 +frame_id: "camera_color_optical_frame" +[INFO] [1733884972.990952]: received mask_obs with header seq: 3938 +stamp: + secs: 1733884972 + nsecs: 994986534 +frame_id: '' and grasped_object_mask header seq: 99089 +stamp: + secs: 1733884972 + nsecs: 783866167 +frame_id: "camera_color_optical_frame" +[INFO] [1733884973.195007]: received mask_obs with header seq: 3939 +stamp: + secs: 1733884973 + nsecs: 199215412 +frame_id: '' and grasped_object_mask header seq: 99095 +stamp: + secs: 1733884972 + nsecs: 983698368 +frame_id: "camera_color_optical_frame" +[INFO] [1733884973.406379]: received mask_obs with header seq: 3940 +stamp: + secs: 1733884973 + nsecs: 410442829 +frame_id: '' and grasped_object_mask header seq: 99101 +stamp: + secs: 1733884973 + nsecs: 183530569 +frame_id: "camera_color_optical_frame" +[INFO] [1733884973.605827]: received mask_obs with header seq: 3941 +stamp: + secs: 1733884973 + nsecs: 609913349 +frame_id: '' and grasped_object_mask header seq: 99107 +stamp: + secs: 1733884973 + nsecs: 383362532 +frame_id: "camera_color_optical_frame" +[INFO] [1733884973.800059]: received mask_obs with header seq: 3942 +stamp: + secs: 1733884973 + nsecs: 804035663 +frame_id: '' and grasped_object_mask header seq: 99113 +stamp: + secs: 1733884973 + nsecs: 583194256 +frame_id: "camera_color_optical_frame" +[INFO] [1733884974.004137]: received mask_obs with header seq: 3943 +stamp: + secs: 1733884974 + nsecs: 8241891 +frame_id: '' and grasped_object_mask header seq: 99119 +stamp: + secs: 1733884973 + nsecs: 783025742 +frame_id: "camera_color_optical_frame" +[INFO] [1733884974.198835]: received mask_obs with header seq: 3944 +stamp: + secs: 1733884974 + nsecs: 202025413 +frame_id: '' and grasped_object_mask header seq: 99125 +stamp: + secs: 1733884973 + nsecs: 982857943 +frame_id: "camera_color_optical_frame" +[INFO] [1733884974.405318]: received mask_obs with header seq: 3945 +stamp: + secs: 1733884974 + nsecs: 409394741 +frame_id: '' and grasped_object_mask header seq: 99131 +stamp: + secs: 1733884974 + nsecs: 182692766 +frame_id: "camera_color_optical_frame" +[INFO] [1733884974.612107]: received mask_obs with header seq: 3946 +stamp: + secs: 1733884974 + nsecs: 616307020 +frame_id: '' and grasped_object_mask header seq: 99137 +stamp: + secs: 1733884974 + nsecs: 382528305 +frame_id: "camera_color_optical_frame" +[INFO] [1733884974.809726]: received mask_obs with header seq: 3947 +stamp: + secs: 1733884974 + nsecs: 813852310 +frame_id: '' and grasped_object_mask header seq: 99143 +stamp: + secs: 1733884974 + nsecs: 582363844 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.003906]: received mask_obs with header seq: 3948 +stamp: + secs: 1733884975 + nsecs: 8115530 +frame_id: '' and grasped_object_mask header seq: 99149 +stamp: + secs: 1733884974 + nsecs: 782199144 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.203026]: received mask_obs with header seq: 3949 +stamp: + secs: 1733884975 + nsecs: 207210779 +frame_id: '' and grasped_object_mask header seq: 99155 +stamp: + secs: 1733884974 + nsecs: 982034445 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.407917]: received mask_obs with header seq: 3950 +stamp: + secs: 1733884975 + nsecs: 411250352 +frame_id: '' and grasped_object_mask header seq: 99161 +stamp: + secs: 1733884975 + nsecs: 181868553 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.602536]: received mask_obs with header seq: 3951 +stamp: + secs: 1733884975 + nsecs: 606728553 +frame_id: '' and grasped_object_mask header seq: 99167 +stamp: + secs: 1733884975 + nsecs: 381701231 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.801763]: received mask_obs with header seq: 3952 +stamp: + secs: 1733884975 + nsecs: 805976629 +frame_id: '' and grasped_object_mask header seq: 99173 +stamp: + secs: 1733884975 + nsecs: 581533194 +frame_id: "camera_color_optical_frame" +[INFO] [1733884975.997902]: received mask_obs with header seq: 3953 +stamp: + secs: 1733884976 + nsecs: 2025365 +frame_id: '' and grasped_object_mask header seq: 99179 +stamp: + secs: 1733884975 + nsecs: 781365156 +frame_id: "camera_color_optical_frame" +[INFO] [1733884976.203087]: received mask_obs with header seq: 3954 +stamp: + secs: 1733884976 + nsecs: 206999540 +frame_id: '' and grasped_object_mask header seq: 99185 +stamp: + secs: 1733884975 + nsecs: 981196880 +frame_id: "camera_color_optical_frame" +[INFO] [1733884976.412903]: received mask_obs with header seq: 3955 +stamp: + secs: 1733884976 + nsecs: 416872739 +frame_id: '' and grasped_object_mask header seq: 99191 +stamp: + secs: 1733884976 + nsecs: 181033850 +frame_id: "camera_color_optical_frame" +[INFO] [1733884976.613189]: received mask_obs with header seq: 3956 +stamp: + secs: 1733884976 + nsecs: 617354869 +frame_id: '' and grasped_object_mask header seq: 99197 +stamp: + secs: 1733884976 + nsecs: 380872965 +frame_id: "camera_color_optical_frame" +[INFO] [1733884976.807168]: received mask_obs with header seq: 3957 +stamp: + secs: 1733884976 + nsecs: 811364889 +frame_id: '' and grasped_object_mask header seq: 99203 +stamp: + secs: 1733884976 + nsecs: 580711365 +frame_id: "camera_color_optical_frame" +[INFO] [1733884976.997593]: received mask_obs with header seq: 3958 +stamp: + secs: 1733884977 + nsecs: 204324 +frame_id: '' and grasped_object_mask header seq: 99209 +stamp: + secs: 1733884976 + nsecs: 780549765 +frame_id: "camera_color_optical_frame" +[INFO] [1733884977.202628]: received mask_obs with header seq: 3959 +stamp: + secs: 1733884977 + nsecs: 206811189 +frame_id: '' and grasped_object_mask header seq: 99215 +stamp: + secs: 1733884976 + nsecs: 980387926 +frame_id: "camera_color_optical_frame" +[INFO] [1733884977.397964]: received mask_obs with header seq: 3960 +stamp: + secs: 1733884977 + nsecs: 401836395 +frame_id: '' and grasped_object_mask header seq: 99221 +stamp: + secs: 1733884977 + nsecs: 180220366 +frame_id: "camera_color_optical_frame" +[INFO] [1733884977.596650]: received mask_obs with header seq: 3961 +stamp: + secs: 1733884977 + nsecs: 600780010 +frame_id: '' and grasped_object_mask header seq: 99227 +stamp: + secs: 1733884977 + nsecs: 380051374 +frame_id: "camera_color_optical_frame" +[INFO] [1733884977.799734]: received mask_obs with header seq: 3962 +stamp: + secs: 1733884977 + nsecs: 803670167 +frame_id: '' and grasped_object_mask header seq: 99233 +stamp: + secs: 1733884977 + nsecs: 579882860 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.002454]: received mask_obs with header seq: 3963 +stamp: + secs: 1733884978 + nsecs: 6635665 +frame_id: '' and grasped_object_mask header seq: 99239 +stamp: + secs: 1733884977 + nsecs: 779713154 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.201345]: received mask_obs with header seq: 3964 +stamp: + secs: 1733884978 + nsecs: 205522060 +frame_id: '' and grasped_object_mask header seq: 99245 +stamp: + secs: 1733884977 + nsecs: 979543209 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.400789]: received mask_obs with header seq: 3965 +stamp: + secs: 1733884978 + nsecs: 404833078 +frame_id: '' and grasped_object_mask header seq: 99251 +stamp: + secs: 1733884978 + nsecs: 179378510 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.599173]: received mask_obs with header seq: 3966 +stamp: + secs: 1733884978 + nsecs: 603354454 +frame_id: '' and grasped_object_mask header seq: 99257 +stamp: + secs: 1733884978 + nsecs: 379214764 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.799019]: received mask_obs with header seq: 3967 +stamp: + secs: 1733884978 + nsecs: 802747726 +frame_id: '' and grasped_object_mask header seq: 99263 +stamp: + secs: 1733884978 + nsecs: 579051018 +frame_id: "camera_color_optical_frame" +[INFO] [1733884978.995086]: received mask_obs with header seq: 3968 +stamp: + secs: 1733884978 + nsecs: 998380661 +frame_id: '' and grasped_object_mask header seq: 99269 +stamp: + secs: 1733884978 + nsecs: 778887272 +frame_id: "camera_color_optical_frame" +[INFO] [1733884979.196118]: received mask_obs with header seq: 3969 +stamp: + secs: 1733884979 + nsecs: 199782609 +frame_id: '' and grasped_object_mask header seq: 99275 +stamp: + secs: 1733884978 + nsecs: 978724003 +frame_id: "camera_color_optical_frame" +[INFO] [1733884979.399731]: received mask_obs with header seq: 3970 +stamp: + secs: 1733884979 + nsecs: 403931617 +frame_id: '' and grasped_object_mask header seq: 99281 +stamp: + secs: 1733884979 + nsecs: 178563595 +frame_id: "camera_color_optical_frame" +[INFO] [1733884979.601490]: received mask_obs with header seq: 3971 +stamp: + secs: 1733884979 + nsecs: 605607986 +frame_id: '' and grasped_object_mask header seq: 99287 +stamp: + secs: 1733884979 + nsecs: 378403664 +frame_id: "camera_color_optical_frame" +[INFO] [1733884979.803455]: received mask_obs with header seq: 3972 +stamp: + secs: 1733884979 + nsecs: 807493686 +frame_id: '' and grasped_object_mask header seq: 99293 +stamp: + secs: 1733884979 + nsecs: 578243732 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.008892]: received mask_obs with header seq: 3973 +stamp: + secs: 1733884980 + nsecs: 13033390 +frame_id: '' and grasped_object_mask header seq: 99299 +stamp: + secs: 1733884979 + nsecs: 778083801 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.205060]: received mask_obs with header seq: 3974 +stamp: + secs: 1733884980 + nsecs: 208876848 +frame_id: '' and grasped_object_mask header seq: 99305 +stamp: + secs: 1733884979 + nsecs: 977924109 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.399727]: received mask_obs with header seq: 3975 +stamp: + secs: 1733884980 + nsecs: 403708219 +frame_id: '' and grasped_object_mask header seq: 99311 +stamp: + secs: 1733884980 + nsecs: 177767754 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.599491]: received mask_obs with header seq: 3976 +stamp: + secs: 1733884980 + nsecs: 603689193 +frame_id: '' and grasped_object_mask header seq: 99317 +stamp: + secs: 1733884980 + nsecs: 377612352 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.793931]: received mask_obs with header seq: 3977 +stamp: + secs: 1733884980 + nsecs: 797293424 +frame_id: '' and grasped_object_mask header seq: 99323 +stamp: + secs: 1733884980 + nsecs: 577456951 +frame_id: "camera_color_optical_frame" +[INFO] [1733884980.993009]: received mask_obs with header seq: 3978 +stamp: + secs: 1733884980 + nsecs: 996627092 +frame_id: '' and grasped_object_mask header seq: 99329 +stamp: + secs: 1733884980 + nsecs: 777302265 +frame_id: "camera_color_optical_frame" +[INFO] [1733884981.194287]: received mask_obs with header seq: 3979 +stamp: + secs: 1733884981 + nsecs: 197581291 +frame_id: '' and grasped_object_mask header seq: 99335 +stamp: + secs: 1733884980 + nsecs: 977147818 +frame_id: "camera_color_optical_frame" +[INFO] [1733884981.398657]: received mask_obs with header seq: 3980 +stamp: + secs: 1733884981 + nsecs: 402729272 +frame_id: '' and grasped_object_mask header seq: 99341 +stamp: + secs: 1733884981 + nsecs: 176986217 +frame_id: "camera_color_optical_frame" +[INFO] [1733884981.593386]: received mask_obs with header seq: 3981 +stamp: + secs: 1733884981 + nsecs: 597202539 +frame_id: '' and grasped_object_mask header seq: 99347 +stamp: + secs: 1733884981 + nsecs: 376823425 +frame_id: "camera_color_optical_frame" +[INFO] [1733884981.804195]: received mask_obs with header seq: 3982 +stamp: + secs: 1733884981 + nsecs: 808274030 +frame_id: '' and grasped_object_mask header seq: 99353 +stamp: + secs: 1733884981 + nsecs: 576659441 +frame_id: "camera_color_optical_frame" +[INFO] [1733884981.997869]: received mask_obs with header seq: 3983 +stamp: + secs: 1733884982 + nsecs: 1746654 +frame_id: '' and grasped_object_mask header seq: 99359 +stamp: + secs: 1733884981 + nsecs: 776495457 +frame_id: "camera_color_optical_frame" +[INFO] [1733884982.196717]: received mask_obs with header seq: 3984 +stamp: + secs: 1733884982 + nsecs: 200887918 +frame_id: '' and grasped_object_mask header seq: 99365 +stamp: + secs: 1733884981 + nsecs: 976330996 +frame_id: "camera_color_optical_frame" +[INFO] [1733884982.405678]: received mask_obs with header seq: 3985 +stamp: + secs: 1733884982 + nsecs: 409892082 +frame_id: '' and grasped_object_mask header seq: 99371 +stamp: + secs: 1733884982 + nsecs: 176169395 +frame_id: "camera_color_optical_frame" +[INFO] [1733884982.605226]: received mask_obs with header seq: 3986 +stamp: + secs: 1733884982 + nsecs: 609401941 +frame_id: '' and grasped_object_mask header seq: 99377 +stamp: + secs: 1733884982 + nsecs: 376008749 +frame_id: "camera_color_optical_frame" +[INFO] [1733884982.796517]: received mask_obs with header seq: 3987 +stamp: + secs: 1733884982 + nsecs: 800484895 +frame_id: '' and grasped_object_mask header seq: 99383 +stamp: + secs: 1733884982 + nsecs: 575849056 +frame_id: "camera_color_optical_frame" +[INFO] [1733884982.994421]: received mask_obs with header seq: 3988 +stamp: + secs: 1733884982 + nsecs: 998512268 +frame_id: '' and grasped_object_mask header seq: 99389 +stamp: + secs: 1733884982 + nsecs: 775688410 +frame_id: "camera_color_optical_frame" +[INFO] [1733884983.196519]: received mask_obs with header seq: 3989 +stamp: + secs: 1733884983 + nsecs: 200722694 +frame_id: '' and grasped_object_mask header seq: 99395 +stamp: + secs: 1733884982 + nsecs: 975528002 +frame_id: "camera_color_optical_frame" +[INFO] [1733884983.401714]: received mask_obs with header seq: 3990 +stamp: + secs: 1733884983 + nsecs: 405913829 +frame_id: '' and grasped_object_mask header seq: 99401 +stamp: + secs: 1733884983 + nsecs: 175366163 +frame_id: "camera_color_optical_frame" +[INFO] [1733884983.608545]: received mask_obs with header seq: 3991 +stamp: + secs: 1733884983 + nsecs: 612523555 +frame_id: '' and grasped_object_mask header seq: 99407 +stamp: + secs: 1733884983 + nsecs: 375203609 +frame_id: "camera_color_optical_frame" +[INFO] [1733884983.804622]: received mask_obs with header seq: 3992 +stamp: + secs: 1733884983 + nsecs: 808704853 +frame_id: '' and grasped_object_mask header seq: 99413 +stamp: + secs: 1733884983 + nsecs: 575041294 +frame_id: "camera_color_optical_frame" +[INFO] [1733884983.995905]: received mask_obs with header seq: 3993 +stamp: + secs: 1733884984 + nsecs: 67949 +frame_id: '' and grasped_object_mask header seq: 99419 +stamp: + secs: 1733884983 + nsecs: 774879932 +frame_id: "camera_color_optical_frame" +[INFO] [1733884984.199848]: received mask_obs with header seq: 3994 +stamp: + secs: 1733884984 + nsecs: 204067945 +frame_id: '' and grasped_object_mask header seq: 99425 +stamp: + secs: 1733884983 + nsecs: 974717379 +frame_id: "camera_color_optical_frame" +[INFO] [1733884984.416605]: received mask_obs with header seq: 3995 +stamp: + secs: 1733884984 + nsecs: 420788049 +frame_id: '' and grasped_object_mask header seq: 99431 +stamp: + secs: 1733884984 + nsecs: 174554110 +frame_id: "camera_color_optical_frame" +[INFO] [1733884984.633225]: received mask_obs with header seq: 3996 +stamp: + secs: 1733884984 + nsecs: 637191772 +frame_id: '' and grasped_object_mask header seq: 99437 +stamp: + secs: 1733884984 + nsecs: 374390364 +frame_id: "camera_color_optical_frame" +[INFO] [1733884984.835769]: received mask_obs with header seq: 3997 +stamp: + secs: 1733884984 + nsecs: 839968204 +frame_id: '' and grasped_object_mask header seq: 99444 +stamp: + secs: 1733884984 + nsecs: 607533216 +frame_id: "camera_color_optical_frame" +[INFO] [1733884985.037260]: received mask_obs with header seq: 3998 +stamp: + secs: 1733884985 + nsecs: 41163921 +frame_id: '' and grasped_object_mask header seq: 99450 +stamp: + secs: 1733884984 + nsecs: 807369471 +frame_id: "camera_color_optical_frame" +[INFO] [1733884985.227837]: received mask_obs with header seq: 3999 +stamp: + secs: 1733884985 + nsecs: 231997013 +frame_id: '' and grasped_object_mask header seq: 99456 +stamp: + secs: 1733884985 + nsecs: 7205486 +frame_id: "camera_color_optical_frame" +[INFO] [1733884985.425626]: received mask_obs with header seq: 4000 +stamp: + secs: 1733884985 + nsecs: 428512096 +frame_id: '' and grasped_object_mask header seq: 99462 +stamp: + secs: 1733884985 + nsecs: 207045794 +frame_id: "camera_color_optical_frame" +[INFO] [1733884985.634147]: received mask_obs with header seq: 4001 +stamp: + secs: 1733884985 + nsecs: 638067245 +frame_id: '' and grasped_object_mask header seq: 99467 +stamp: + secs: 1733884985 + nsecs: 373580456 +frame_id: "camera_color_optical_frame" +[INFO] [1733884985.842181]: received mask_obs with header seq: 4002 +stamp: + secs: 1733884985 + nsecs: 846258878 +frame_id: '' and grasped_object_mask header seq: 99474 +stamp: + secs: 1733884985 + nsecs: 606729507 +frame_id: "camera_color_optical_frame" +[INFO] [1733884986.026330]: received mask_obs with header seq: 4003 +stamp: + secs: 1733884986 + nsecs: 29599905 +frame_id: '' and grasped_object_mask header seq: 99480 +stamp: + secs: 1733884985 + nsecs: 806571007 +frame_id: "camera_color_optical_frame" +[INFO] [1733884986.226847]: received mask_obs with header seq: 4004 +stamp: + secs: 1733884986 + nsecs: 230706453 +frame_id: '' and grasped_object_mask header seq: 99486 +stamp: + secs: 1733884986 + nsecs: 6412745 +frame_id: "camera_color_optical_frame" +[INFO] [1733884986.438918]: received mask_obs with header seq: 4005 +stamp: + secs: 1733884986 + nsecs: 443017959 +frame_id: '' and grasped_object_mask header seq: 99492 +stamp: + secs: 1733884986 + nsecs: 206249475 +frame_id: "camera_color_optical_frame" +[INFO] [1733884986.636394]: received mask_obs with header seq: 4006 +stamp: + secs: 1733884986 + nsecs: 640626430 +frame_id: '' and grasped_object_mask header seq: 99498 +stamp: + secs: 1733884986 + nsecs: 406085491 +frame_id: "camera_color_optical_frame" +[INFO] [1733884986.827130]: received mask_obs with header seq: 4007 +stamp: + secs: 1733884986 + nsecs: 830031156 +frame_id: '' and grasped_object_mask header seq: 99504 +stamp: + secs: 1733884986 + nsecs: 605921268 +frame_id: "camera_color_optical_frame" +[INFO] [1733884987.036192]: received mask_obs with header seq: 4008 +stamp: + secs: 1733884987 + nsecs: 40397167 +frame_id: '' and grasped_object_mask header seq: 99510 +stamp: + secs: 1733884986 + nsecs: 805757046 +frame_id: "camera_color_optical_frame" +[INFO] [1733884987.239697]: received mask_obs with header seq: 4009 +stamp: + secs: 1733884987 + nsecs: 243845224 +frame_id: '' and grasped_object_mask header seq: 99516 +stamp: + secs: 1733884987 + nsecs: 5594015 +frame_id: "camera_color_optical_frame" +[INFO] [1733884987.426261]: received mask_obs with header seq: 4010 +stamp: + secs: 1733884987 + nsecs: 430520534 +frame_id: '' and grasped_object_mask header seq: 99522 +stamp: + secs: 1733884987 + nsecs: 205438852 +frame_id: "camera_color_optical_frame" +[INFO] [1733884987.637936]: received mask_obs with header seq: 4011 +stamp: + secs: 1733884987 + nsecs: 642069339 +frame_id: '' and grasped_object_mask header seq: 99528 +stamp: + secs: 1733884987 + nsecs: 405285597 +frame_id: "camera_color_optical_frame" +[INFO] [1733884987.837276]: received mask_obs with header seq: 4012 +stamp: + secs: 1733884987 + nsecs: 841227531 +frame_id: '' and grasped_object_mask header seq: 99534 +stamp: + secs: 1733884987 + nsecs: 605132818 +frame_id: "camera_color_optical_frame" +[INFO] [1733884988.032785]: received mask_obs with header seq: 4013 +stamp: + secs: 1733884988 + nsecs: 37025213 +frame_id: '' and grasped_object_mask header seq: 99540 +stamp: + secs: 1733884987 + nsecs: 804980516 +frame_id: "camera_color_optical_frame" +[INFO] [1733884988.229748]: received mask_obs with header seq: 4014 +stamp: + secs: 1733884988 + nsecs: 232890844 +frame_id: '' and grasped_object_mask header seq: 99546 +stamp: + secs: 1733884988 + nsecs: 4828453 +frame_id: "camera_color_optical_frame" +[INFO] [1733884988.426575]: received mask_obs with header seq: 4015 +stamp: + secs: 1733884988 + nsecs: 429852485 +frame_id: '' and grasped_object_mask header seq: 99552 +stamp: + secs: 1733884988 + nsecs: 204670668 +frame_id: "camera_color_optical_frame" +[INFO] [1733884988.625890]: received mask_obs with header seq: 4016 +stamp: + secs: 1733884988 + nsecs: 629469633 +frame_id: '' and grasped_object_mask header seq: 99558 +stamp: + secs: 1733884988 + nsecs: 404510975 +frame_id: "camera_color_optical_frame" +[INFO] [1733884988.820274]: received mask_obs with header seq: 4017 +stamp: + secs: 1733884988 + nsecs: 822649955 +frame_id: '' and grasped_object_mask header seq: 99563 +stamp: + secs: 1733884988 + nsecs: 571044207 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.003924]: received mask_obs with header seq: 4018 +stamp: + secs: 1733884989 + nsecs: 7812976 +frame_id: '' and grasped_object_mask header seq: 99569 +stamp: + secs: 1733884988 + nsecs: 770885706 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.198515]: received mask_obs with header seq: 4019 +stamp: + secs: 1733884989 + nsecs: 202775955 +frame_id: '' and grasped_object_mask header seq: 99575 +stamp: + secs: 1733884988 + nsecs: 970726252 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.398352]: received mask_obs with header seq: 4020 +stamp: + secs: 1733884989 + nsecs: 402585744 +frame_id: '' and grasped_object_mask header seq: 99581 +stamp: + secs: 1733884989 + nsecs: 170566082 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.603428]: received mask_obs with header seq: 4021 +stamp: + secs: 1733884989 + nsecs: 607581377 +frame_id: '' and grasped_object_mask header seq: 99587 +stamp: + secs: 1733884989 + nsecs: 370405436 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.802189]: received mask_obs with header seq: 4022 +stamp: + secs: 1733884989 + nsecs: 806339263 +frame_id: '' and grasped_object_mask header seq: 99593 +stamp: + secs: 1733884989 + nsecs: 570245028 +frame_id: "camera_color_optical_frame" +[INFO] [1733884989.996156]: received mask_obs with header seq: 4023 +stamp: + secs: 1733884990 + nsecs: 18358 +frame_id: '' and grasped_object_mask header seq: 99599 +stamp: + secs: 1733884989 + nsecs: 770084381 +frame_id: "camera_color_optical_frame" +[INFO] [1733884990.193342]: received mask_obs with header seq: 4024 +stamp: + secs: 1733884990 + nsecs: 197146892 +frame_id: '' and grasped_object_mask header seq: 99605 +stamp: + secs: 1733884989 + nsecs: 969925165 +frame_id: "camera_color_optical_frame" +[INFO] [1733884990.397266]: received mask_obs with header seq: 4025 +stamp: + secs: 1733884990 + nsecs: 401518583 +frame_id: '' and grasped_object_mask header seq: 99611 +stamp: + secs: 1733884990 + nsecs: 169763327 +frame_id: "camera_color_optical_frame" +[INFO] [1733884990.601821]: received mask_obs with header seq: 4026 +stamp: + secs: 1733884990 + nsecs: 605793714 +frame_id: '' and grasped_object_mask header seq: 99617 +stamp: + secs: 1733884990 + nsecs: 369601250 +frame_id: "camera_color_optical_frame" +[INFO] [1733884990.799382]: received mask_obs with header seq: 4027 +stamp: + secs: 1733884990 + nsecs: 803618192 +frame_id: '' and grasped_object_mask header seq: 99623 +stamp: + secs: 1733884990 + nsecs: 569439173 +frame_id: "camera_color_optical_frame" +[INFO] [1733884990.998338]: received mask_obs with header seq: 4028 +stamp: + secs: 1733884991 + nsecs: 2559423 +frame_id: '' and grasped_object_mask header seq: 99629 +stamp: + secs: 1733884990 + nsecs: 769277096 +frame_id: "camera_color_optical_frame" +[INFO] [1733884991.193652]: received mask_obs with header seq: 4029 +stamp: + secs: 1733884991 + nsecs: 197855710 +frame_id: '' and grasped_object_mask header seq: 99635 +stamp: + secs: 1733884990 + nsecs: 969115019 +frame_id: "camera_color_optical_frame" +[INFO] [1733884991.400006]: received mask_obs with header seq: 4030 +stamp: + secs: 1733884991 + nsecs: 403799057 +frame_id: '' and grasped_object_mask header seq: 99641 +stamp: + secs: 1733884991 + nsecs: 168953419 +frame_id: "camera_color_optical_frame" +[INFO] [1733884991.595814]: received mask_obs with header seq: 4031 +stamp: + secs: 1733884991 + nsecs: 600054979 +frame_id: '' and grasped_object_mask header seq: 99647 +stamp: + secs: 1733884991 + nsecs: 368790388 +frame_id: "camera_color_optical_frame" +[INFO] [1733884991.799621]: received mask_obs with header seq: 4032 +stamp: + secs: 1733884991 + nsecs: 803595066 +frame_id: '' and grasped_object_mask header seq: 99653 +stamp: + secs: 1733884991 + nsecs: 568626881 +frame_id: "camera_color_optical_frame" +[INFO] [1733884991.999880]: received mask_obs with header seq: 4033 +stamp: + secs: 1733884992 + nsecs: 4093408 +frame_id: '' and grasped_object_mask header seq: 99659 +stamp: + secs: 1733884991 + nsecs: 768463612 +frame_id: "camera_color_optical_frame" +[INFO] [1733884992.206579]: received mask_obs with header seq: 4034 +stamp: + secs: 1733884992 + nsecs: 210733890 +frame_id: '' and grasped_object_mask header seq: 99665 +stamp: + secs: 1733884991 + nsecs: 968300343 +frame_id: "camera_color_optical_frame" +[INFO] [1733884992.399710]: received mask_obs with header seq: 4035 +stamp: + secs: 1733884992 + nsecs: 403930902 +frame_id: '' and grasped_object_mask header seq: 99671 +stamp: + secs: 1733884992 + nsecs: 168140650 +frame_id: "camera_color_optical_frame" +[INFO] [1733884992.600813]: received mask_obs with header seq: 4036 +stamp: + secs: 1733884992 + nsecs: 604932308 +frame_id: '' and grasped_object_mask header seq: 99677 +stamp: + secs: 1733884992 + nsecs: 367984295 +frame_id: "camera_color_optical_frame" +[INFO] [1733884992.802453]: received mask_obs with header seq: 4037 +stamp: + secs: 1733884992 + nsecs: 806342601 +frame_id: '' and grasped_object_mask header seq: 99683 +stamp: + secs: 1733884992 + nsecs: 567826986 +frame_id: "camera_color_optical_frame" +[INFO] [1733884992.998813]: received mask_obs with header seq: 4038 +stamp: + secs: 1733884993 + nsecs: 2985715 +frame_id: '' and grasped_object_mask header seq: 99689 +stamp: + secs: 1733884992 + nsecs: 767669916 +frame_id: "camera_color_optical_frame" +[INFO] [1733884993.203711]: received mask_obs with header seq: 4039 +stamp: + secs: 1733884993 + nsecs: 207891464 +frame_id: '' and grasped_object_mask header seq: 99695 +stamp: + secs: 1733884992 + nsecs: 967512846 +frame_id: "camera_color_optical_frame" +[INFO] [1733884993.398866]: received mask_obs with header seq: 4040 +stamp: + secs: 1733884993 + nsecs: 402997732 +frame_id: '' and grasped_object_mask header seq: 99701 +stamp: + secs: 1733884993 + nsecs: 167351007 +frame_id: "camera_color_optical_frame" +[INFO] [1733884993.607717]: received mask_obs with header seq: 4041 +stamp: + secs: 1733884993 + nsecs: 611915588 +frame_id: '' and grasped_object_mask header seq: 99707 +stamp: + secs: 1733884993 + nsecs: 367186785 +frame_id: "camera_color_optical_frame" +[INFO] [1733884993.804632]: received mask_obs with header seq: 4042 +stamp: + secs: 1733884993 + nsecs: 808839797 +frame_id: '' and grasped_object_mask header seq: 99713 +stamp: + secs: 1733884993 + nsecs: 567022085 +frame_id: "camera_color_optical_frame" +[INFO] [1733884994.009596]: received mask_obs with header seq: 4043 +stamp: + secs: 1733884994 + nsecs: 13694524 +frame_id: '' and grasped_object_mask header seq: 99719 +stamp: + secs: 1733884993 + nsecs: 766858101 +frame_id: "camera_color_optical_frame" +[INFO] [1733884994.208955]: received mask_obs with header seq: 4044 +stamp: + secs: 1733884994 + nsecs: 213088274 +frame_id: '' and grasped_object_mask header seq: 99725 +stamp: + secs: 1733884993 + nsecs: 966693163 +frame_id: "camera_color_optical_frame" +[INFO] [1733884994.404317]: received mask_obs with header seq: 4045 +stamp: + secs: 1733884994 + nsecs: 408331632 +frame_id: '' and grasped_object_mask header seq: 99731 +stamp: + secs: 1733884994 + nsecs: 166528940 +frame_id: "camera_color_optical_frame" +[INFO] [1733884994.616458]: received mask_obs with header seq: 4046 +stamp: + secs: 1733884994 + nsecs: 620616197 +frame_id: '' and grasped_object_mask header seq: 99737 +stamp: + secs: 1733884994 + nsecs: 366365433 +frame_id: "camera_color_optical_frame" +[INFO] [1733884994.819554]: received mask_obs with header seq: 4047 +stamp: + secs: 1733884994 + nsecs: 823729038 +frame_id: '' and grasped_object_mask header seq: 99744 +stamp: + secs: 1733884994 + nsecs: 599508286 +frame_id: "camera_color_optical_frame" +[INFO] [1733884995.020024]: received mask_obs with header seq: 4048 +stamp: + secs: 1733884995 + nsecs: 24137496 +frame_id: '' and grasped_object_mask header seq: 99749 +stamp: + secs: 1733884994 + nsecs: 766038179 +frame_id: "camera_color_optical_frame" +[INFO] [1733884995.222428]: received mask_obs with header seq: 4049 +stamp: + secs: 1733884995 + nsecs: 226561069 +frame_id: '' and grasped_object_mask header seq: 99755 +stamp: + secs: 1733884994 + nsecs: 965875149 +frame_id: "camera_color_optical_frame" +[INFO] [1733884995.430377]: received mask_obs with header seq: 4050 +stamp: + secs: 1733884995 + nsecs: 434581995 +frame_id: '' and grasped_object_mask header seq: 99762 +stamp: + secs: 1733884995 + nsecs: 199022770 +frame_id: "camera_color_optical_frame" +[INFO] [1733884995.620886]: received mask_obs with header seq: 4051 +stamp: + secs: 1733884995 + nsecs: 625000953 +frame_id: '' and grasped_object_mask header seq: 99768 +stamp: + secs: 1733884995 + nsecs: 398865938 +frame_id: "camera_color_optical_frame" +[INFO] [1733884995.817193]: received mask_obs with header seq: 4052 +stamp: + secs: 1733884995 + nsecs: 821452617 +frame_id: '' and grasped_object_mask header seq: 99774 +stamp: + secs: 1733884995 + nsecs: 598710299 +frame_id: "camera_color_optical_frame" +[INFO] [1733884996.019032]: received mask_obs with header seq: 4053 +stamp: + secs: 1733884996 + nsecs: 23197412 +frame_id: '' and grasped_object_mask header seq: 99780 +stamp: + secs: 1733884995 + nsecs: 798553944 +frame_id: "camera_color_optical_frame" +[INFO] [1733884996.218394]: received mask_obs with header seq: 4054 +stamp: + secs: 1733884996 + nsecs: 222412109 +frame_id: '' and grasped_object_mask header seq: 99786 +stamp: + secs: 1733884995 + nsecs: 998397589 +frame_id: "camera_color_optical_frame" +[INFO] [1733884996.420883]: received mask_obs with header seq: 4055 +stamp: + secs: 1733884996 + nsecs: 424916267 +frame_id: '' and grasped_object_mask header seq: 99792 +stamp: + secs: 1733884996 + nsecs: 199400902 +frame_id: "camera_color_optical_frame" +[INFO] [1733884996.611526]: received mask_obs with header seq: 4056 +stamp: + secs: 1733884996 + nsecs: 615666389 +frame_id: '' and grasped_object_mask header seq: 99797 +stamp: + secs: 1733884996 + nsecs: 367902517 +frame_id: "camera_color_optical_frame" +[INFO] [1733884996.804754]: received mask_obs with header seq: 4057 +stamp: + secs: 1733884996 + nsecs: 808830738 +frame_id: '' and grasped_object_mask header seq: 99803 +stamp: + secs: 1733884996 + nsecs: 570184469 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.014226]: received mask_obs with header seq: 4058 +stamp: + secs: 1733884997 + nsecs: 18452405 +frame_id: '' and grasped_object_mask header seq: 99809 +stamp: + secs: 1733884996 + nsecs: 772554636 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.213196]: received mask_obs with header seq: 4059 +stamp: + secs: 1733884997 + nsecs: 216432809 +frame_id: '' and grasped_object_mask header seq: 99816 +stamp: + secs: 1733884997 + nsecs: 8764982 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.420022]: received mask_obs with header seq: 4060 +stamp: + secs: 1733884997 + nsecs: 424243211 +frame_id: '' and grasped_object_mask header seq: 99822 +stamp: + secs: 1733884997 + nsecs: 209839106 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.603607]: received mask_obs with header seq: 4061 +stamp: + secs: 1733884997 + nsecs: 607669830 +frame_id: '' and grasped_object_mask header seq: 99827 +stamp: + secs: 1733884997 + nsecs: 376185894 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.800593]: received mask_obs with header seq: 4062 +stamp: + secs: 1733884997 + nsecs: 804829120 +frame_id: '' and grasped_object_mask header seq: 99833 +stamp: + secs: 1733884997 + nsecs: 575790167 +frame_id: "camera_color_optical_frame" +[INFO] [1733884997.996605]: received mask_obs with header seq: 4063 +stamp: + secs: 1733884998 + nsecs: 805139 +frame_id: '' and grasped_object_mask header seq: 99839 +stamp: + secs: 1733884997 + nsecs: 775383472 +frame_id: "camera_color_optical_frame" +[INFO] [1733884998.196324]: received mask_obs with header seq: 4064 +stamp: + secs: 1733884998 + nsecs: 200482368 +frame_id: '' and grasped_object_mask header seq: 99845 +stamp: + secs: 1733884997 + nsecs: 974965096 +frame_id: "camera_color_optical_frame" +[INFO] [1733884998.421011]: received mask_obs with header seq: 4065 +stamp: + secs: 1733884998 + nsecs: 424982070 +frame_id: '' and grasped_object_mask header seq: 99851 +stamp: + secs: 1733884998 + nsecs: 174552202 +frame_id: "camera_color_optical_frame" +[INFO] [1733884998.617881]: received mask_obs with header seq: 4066 +stamp: + secs: 1733884998 + nsecs: 621931314 +frame_id: '' and grasped_object_mask header seq: 99858 +stamp: + secs: 1733884998 + nsecs: 407429457 +frame_id: "camera_color_optical_frame" +[INFO] [1733884998.822101]: received mask_obs with header seq: 4067 +stamp: + secs: 1733884998 + nsecs: 826096296 +frame_id: '' and grasped_object_mask header seq: 99864 +stamp: + secs: 1733884998 + nsecs: 607025385 +frame_id: "camera_color_optical_frame" +[INFO] [1733884999.018649]: received mask_obs with header seq: 4068 +stamp: + secs: 1733884999 + nsecs: 22788524 +frame_id: '' and grasped_object_mask header seq: 99870 +stamp: + secs: 1733884998 + nsecs: 806609631 +frame_id: "camera_color_optical_frame" +[INFO] [1733884999.060238]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1733884999.204076]: received mask_obs with header seq: 4069 +stamp: + secs: 1733884999 + nsecs: 208339691 +frame_id: '' and grasped_object_mask header seq: 99875 +stamp: + secs: 1733884998 + nsecs: 972921610 +frame_id: "camera_color_optical_frame" +[INFO] [1733884999.820353]: Stored episode 3. +[INFO] [1733884999.821264]: Saving video... +[INFO] [1733885000.374097]: Video saved! +Starting episode 3 +[INFO] [1733885000.410567]: resetting environment +[INFO] [1733885000.431314]: cleared current plan +[INFO] [1733885000.434821]: moving to home +[WARN] [1733885000.384063]: Plan exhausted +[WARN] [1733885000.424048]: Plan exhausted +[INFO] [1733885003.239462]: reached home +[INFO] [1733885003.246721]: reset action history +[INFO] [1733885005.976352]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733885009.653190]: resetting environment +[INFO] [1733885009.660966]: cleared current plan +[INFO] [1733885009.661894]: moving to home +[INFO] [1733885010.665150]: reached home +[INFO] [1733885010.669759]: reset action history +[INFO] [1733885013.400813]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/episode_rosbags/episode_3_2024-12-10-21-43-33.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-608694.log +started roslaunch server http://158.130.50.37:41145/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [609430] +process[print_text-11]: started with pid [609431] +process[rosbag_record-12]: started with pid [609455] +started bagging! +[INFO] [1733885014.074764]: received mask_obs with header seq: 4070 +stamp: + secs: 1733885014 + nsecs: 78712940 +frame_id: '' and grasped_object_mask header seq: 100323 +stamp: + secs: 1733885013 + nsecs: 883937120 +frame_id: "camera_color_optical_frame" +[INFO] [1733885014.307650]: received mask_obs with header seq: 4071 +stamp: + secs: 1733885014 + nsecs: 311574459 +frame_id: '' and grasped_object_mask header seq: 100329 +stamp: + secs: 1733885014 + nsecs: 83785057 +frame_id: "camera_color_optical_frame" +[INFO] [1733885014.496784]: received mask_obs with header seq: 4072 +stamp: + secs: 1733885014 + nsecs: 500941753 +frame_id: '' and grasped_object_mask header seq: 100335 +stamp: + secs: 1733885014 + nsecs: 283630848 +frame_id: "camera_color_optical_frame" +[INFO] [1733885014.707516]: received mask_obs with header seq: 4073 +stamp: + secs: 1733885014 + nsecs: 711723089 +frame_id: '' and grasped_object_mask header seq: 100341 +stamp: + secs: 1733885014 + nsecs: 483476877 +frame_id: "camera_color_optical_frame" +[INFO] [1733885014.908588]: received mask_obs with header seq: 4074 +stamp: + secs: 1733885014 + nsecs: 912441253 +frame_id: '' and grasped_object_mask header seq: 100347 +stamp: + secs: 1733885014 + nsecs: 683324337 +frame_id: "camera_color_optical_frame" +[INFO] [1733885015.097380]: received mask_obs with header seq: 4075 +stamp: + secs: 1733885015 + nsecs: 101612329 +frame_id: '' and grasped_object_mask header seq: 100353 +stamp: + secs: 1733885014 + nsecs: 883170843 +frame_id: "camera_color_optical_frame" +[INFO] [1733885015.301129]: received mask_obs with header seq: 4076 +stamp: + secs: 1733885015 + nsecs: 304673433 +frame_id: '' and grasped_object_mask header seq: 100359 +stamp: + secs: 1733885015 + nsecs: 83018303 +frame_id: "camera_color_optical_frame" +[INFO] [1733885015.517574]: received mask_obs with header seq: 4077 +stamp: + secs: 1733885015 + nsecs: 521790504 +frame_id: '' and grasped_object_mask header seq: 100365 +stamp: + secs: 1733885015 + nsecs: 282858372 +frame_id: "camera_color_optical_frame" +[INFO] [1733885015.714588]: received mask_obs with header seq: 4078 +stamp: + secs: 1733885015 + nsecs: 718834877 +frame_id: '' and grasped_object_mask header seq: 100371 +stamp: + secs: 1733885015 + nsecs: 482698202 +frame_id: "camera_color_optical_frame" +[INFO] [1733885015.910100]: received mask_obs with header seq: 4079 +stamp: + secs: 1733885015 + nsecs: 914264202 +frame_id: '' and grasped_object_mask header seq: 100377 +stamp: + secs: 1733885015 + nsecs: 682538033 +frame_id: "camera_color_optical_frame" +[INFO] [1733885016.121979]: received mask_obs with header seq: 4080 +stamp: + secs: 1733885016 + nsecs: 126222133 +frame_id: '' and grasped_object_mask header seq: 100383 +stamp: + secs: 1733885015 + nsecs: 882378817 +frame_id: "camera_color_optical_frame" +[INFO] [1733885016.332588]: received mask_obs with header seq: 4081 +stamp: + secs: 1733885016 + nsecs: 336833000 +frame_id: '' and grasped_object_mask header seq: 100389 +stamp: + secs: 1733885016 + nsecs: 82218409 +frame_id: "camera_color_optical_frame" +[INFO] [1733885016.533317]: received mask_obs with header seq: 4082 +stamp: + secs: 1733885016 + nsecs: 537486553 +frame_id: '' and grasped_object_mask header seq: 100395 +stamp: + secs: 1733885016 + nsecs: 282053709 +frame_id: "camera_color_optical_frame" +[INFO] [1733885016.722154]: received mask_obs with header seq: 4083 +stamp: + secs: 1733885016 + nsecs: 726374864 +frame_id: '' and grasped_object_mask header seq: 100401 +stamp: + secs: 1733885016 + nsecs: 481888294 +frame_id: "camera_color_optical_frame" +[INFO] [1733885016.926172]: received mask_obs with header seq: 4084 +stamp: + secs: 1733885016 + nsecs: 930395126 +frame_id: '' and grasped_object_mask header seq: 100407 +stamp: + secs: 1733885016 + nsecs: 681722641 +frame_id: "camera_color_optical_frame" +[INFO] [1733885017.115224]: received mask_obs with header seq: 4085 +stamp: + secs: 1733885017 + nsecs: 119508981 +frame_id: '' and grasped_object_mask header seq: 100413 +stamp: + secs: 1733885016 + nsecs: 881556988 +frame_id: "camera_color_optical_frame" +[INFO] [1733885017.308915]: received mask_obs with header seq: 4086 +stamp: + secs: 1733885017 + nsecs: 313059091 +frame_id: '' and grasped_object_mask header seq: 100419 +stamp: + secs: 1733885017 + nsecs: 81391811 +frame_id: "camera_color_optical_frame" +[INFO] [1733885017.513788]: received mask_obs with header seq: 4087 +stamp: + secs: 1733885017 + nsecs: 517732143 +frame_id: '' and grasped_object_mask header seq: 100425 +stamp: + secs: 1733885017 + nsecs: 281231642 +frame_id: "camera_color_optical_frame" +[INFO] [1733885017.707238]: received mask_obs with header seq: 4088 +stamp: + secs: 1733885017 + nsecs: 711510896 +frame_id: '' and grasped_object_mask header seq: 100431 +stamp: + secs: 1733885017 + nsecs: 481071711 +frame_id: "camera_color_optical_frame" +[INFO] [1733885017.918364]: received mask_obs with header seq: 4089 +stamp: + secs: 1733885017 + nsecs: 922608613 +frame_id: '' and grasped_object_mask header seq: 100437 +stamp: + secs: 1733885017 + nsecs: 680911541 +frame_id: "camera_color_optical_frame" +[INFO] [1733885018.110248]: received mask_obs with header seq: 4090 +stamp: + secs: 1733885018 + nsecs: 114225149 +frame_id: '' and grasped_object_mask header seq: 100443 +stamp: + secs: 1733885017 + nsecs: 880751610 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -244953145 for nearest: 1733885017835638295 - target: 1733885018080591440 at idx 59 +[INFO] [1733885018.308580]: received mask_obs with header seq: 4091 +stamp: + secs: 1733885018 + nsecs: 312823772 +frame_id: '' and grasped_object_mask header seq: 100449 +stamp: + secs: 1733885018 + nsecs: 80591440 +frame_id: "camera_color_optical_frame" +[INFO] [1733885018.516400]: received mask_obs with header seq: 4092 +stamp: + secs: 1733885018 + nsecs: 520658969 +frame_id: '' and grasped_object_mask header seq: 100455 +stamp: + secs: 1733885018 + nsecs: 280435562 +frame_id: "camera_color_optical_frame" +[INFO] [1733885018.708810]: received mask_obs with header seq: 4093 +stamp: + secs: 1733885018 + nsecs: 713054895 +frame_id: '' and grasped_object_mask header seq: 100461 +stamp: + secs: 1733885018 + nsecs: 480278730 +frame_id: "camera_color_optical_frame" +[INFO] [1733885018.914128]: received mask_obs with header seq: 4094 +stamp: + secs: 1733885018 + nsecs: 917916774 +frame_id: '' and grasped_object_mask header seq: 100467 +stamp: + secs: 1733885018 + nsecs: 680122137 +frame_id: "camera_color_optical_frame" +[INFO] [1733885019.116379]: received mask_obs with header seq: 4095 +stamp: + secs: 1733885019 + nsecs: 120525598 +frame_id: '' and grasped_object_mask header seq: 100473 +stamp: + secs: 1733885018 + nsecs: 879965544 +frame_id: "camera_color_optical_frame" +[INFO] [1733885019.319450]: received mask_obs with header seq: 4096 +stamp: + secs: 1733885019 + nsecs: 323395252 +frame_id: '' and grasped_object_mask header seq: 100479 +stamp: + secs: 1733885019 + nsecs: 79809189 +frame_id: "camera_color_optical_frame" +[INFO] [1733885019.516478]: received mask_obs with header seq: 4097 +stamp: + secs: 1733885019 + nsecs: 520642042 +frame_id: '' and grasped_object_mask header seq: 100485 +stamp: + secs: 1733885019 + nsecs: 279648542 +frame_id: "camera_color_optical_frame" +[INFO] [1733885019.710253]: received mask_obs with header seq: 4098 +stamp: + secs: 1733885019 + nsecs: 713438749 +frame_id: '' and grasped_object_mask header seq: 100491 +stamp: + secs: 1733885019 + nsecs: 479487658 +frame_id: "camera_color_optical_frame" +[INFO] [1733885019.908374]: received mask_obs with header seq: 4099 +stamp: + secs: 1733885019 + nsecs: 912383794 +frame_id: '' and grasped_object_mask header seq: 100497 +stamp: + secs: 1733885019 + nsecs: 679327726 +frame_id: "camera_color_optical_frame" +[INFO] [1733885020.110398]: received mask_obs with header seq: 4100 +stamp: + secs: 1733885020 + nsecs: 114586114 +frame_id: '' and grasped_object_mask header seq: 100503 +stamp: + secs: 1733885019 + nsecs: 879166842 +frame_id: "camera_color_optical_frame" +[INFO] [1733885020.307503]: received mask_obs with header seq: 4101 +stamp: + secs: 1733885020 + nsecs: 311718940 +frame_id: '' and grasped_object_mask header seq: 100509 +stamp: + secs: 1733885020 + nsecs: 79005957 +frame_id: "camera_color_optical_frame" +[INFO] [1733885020.508269]: received mask_obs with header seq: 4102 +stamp: + secs: 1733885020 + nsecs: 512487411 +frame_id: '' and grasped_object_mask header seq: 100515 +stamp: + secs: 1733885020 + nsecs: 278844833 +frame_id: "camera_color_optical_frame" +[INFO] [1733885020.706459]: received mask_obs with header seq: 4103 +stamp: + secs: 1733885020 + nsecs: 710686922 +frame_id: '' and grasped_object_mask header seq: 100521 +stamp: + secs: 1733885020 + nsecs: 478683710 +frame_id: "camera_color_optical_frame" +[INFO] [1733885020.912189]: received mask_obs with header seq: 4104 +stamp: + secs: 1733885020 + nsecs: 916399955 +frame_id: '' and grasped_object_mask header seq: 100527 +stamp: + secs: 1733885020 + nsecs: 678522348 +frame_id: "camera_color_optical_frame" +[INFO] [1733885021.108885]: received mask_obs with header seq: 4105 +stamp: + secs: 1733885021 + nsecs: 113110542 +frame_id: '' and grasped_object_mask header seq: 100533 +stamp: + secs: 1733885020 + nsecs: 878362417 +frame_id: "camera_color_optical_frame" +[INFO] [1733885021.308839]: received mask_obs with header seq: 4106 +stamp: + secs: 1733885021 + nsecs: 313066005 +frame_id: '' and grasped_object_mask header seq: 100539 +stamp: + secs: 1733885021 + nsecs: 78201056 +frame_id: "camera_color_optical_frame" +[INFO] [1733885021.517854]: received mask_obs with header seq: 4107 +stamp: + secs: 1733885021 + nsecs: 522028446 +frame_id: '' and grasped_object_mask header seq: 100545 +stamp: + secs: 1733885021 + nsecs: 278028965 +frame_id: "camera_color_optical_frame" +[INFO] [1733885021.709568]: received mask_obs with header seq: 4108 +stamp: + secs: 1733885021 + nsecs: 713296175 +frame_id: '' and grasped_object_mask header seq: 100551 +stamp: + secs: 1733885021 + nsecs: 477855444 +frame_id: "camera_color_optical_frame" +[INFO] [1733885021.913963]: received mask_obs with header seq: 4109 +stamp: + secs: 1733885021 + nsecs: 918186187 +frame_id: '' and grasped_object_mask header seq: 100557 +stamp: + secs: 1733885021 + nsecs: 677681684 +frame_id: "camera_color_optical_frame" +[INFO] [1733885022.110898]: received mask_obs with header seq: 4110 +stamp: + secs: 1733885022 + nsecs: 114788293 +frame_id: '' and grasped_object_mask header seq: 100563 +stamp: + secs: 1733885021 + nsecs: 877506733 +frame_id: "camera_color_optical_frame" +[INFO] [1733885022.312001]: received mask_obs with header seq: 4111 +stamp: + secs: 1733885022 + nsecs: 316204071 +frame_id: '' and grasped_object_mask header seq: 100569 +stamp: + secs: 1733885022 + nsecs: 77332735 +frame_id: "camera_color_optical_frame" +[INFO] [1733885022.517516]: received mask_obs with header seq: 4112 +stamp: + secs: 1733885022 + nsecs: 521677970 +frame_id: '' and grasped_object_mask header seq: 100575 +stamp: + secs: 1733885022 + nsecs: 277173758 +frame_id: "camera_color_optical_frame" +[INFO] [1733885022.707579]: received mask_obs with header seq: 4113 +stamp: + secs: 1733885022 + nsecs: 711771249 +frame_id: '' and grasped_object_mask header seq: 100581 +stamp: + secs: 1733885022 + nsecs: 477016687 +frame_id: "camera_color_optical_frame" +[INFO] [1733885022.921558]: received mask_obs with header seq: 4114 +stamp: + secs: 1733885022 + nsecs: 925777196 +frame_id: '' and grasped_object_mask header seq: 100587 +stamp: + secs: 1733885022 + nsecs: 676859140 +frame_id: "camera_color_optical_frame" +[INFO] [1733885023.112077]: received mask_obs with header seq: 4115 +stamp: + secs: 1733885023 + nsecs: 115959882 +frame_id: '' and grasped_object_mask header seq: 100593 +stamp: + secs: 1733885022 + nsecs: 876702070 +frame_id: "camera_color_optical_frame" +[INFO] [1733885023.318284]: received mask_obs with header seq: 4116 +stamp: + secs: 1733885023 + nsecs: 322398900 +frame_id: '' and grasped_object_mask header seq: 100599 +stamp: + secs: 1733885023 + nsecs: 76545238 +frame_id: "camera_color_optical_frame" +[INFO] [1733885023.509864]: received mask_obs with header seq: 4117 +stamp: + secs: 1733885023 + nsecs: 513746976 +frame_id: '' and grasped_object_mask header seq: 100605 +stamp: + secs: 1733885023 + nsecs: 276386976 +frame_id: "camera_color_optical_frame" +[INFO] [1733885023.722829]: received mask_obs with header seq: 4118 +stamp: + secs: 1733885023 + nsecs: 727022647 +frame_id: '' and grasped_object_mask header seq: 100611 +stamp: + secs: 1733885023 + nsecs: 476227760 +frame_id: "camera_color_optical_frame" +[INFO] [1733885023.934497]: received mask_obs with header seq: 4119 +stamp: + secs: 1733885023 + nsecs: 938079833 +frame_id: '' and grasped_object_mask header seq: 100617 +stamp: + secs: 1733885023 + nsecs: 676068544 +frame_id: "camera_color_optical_frame" +[INFO] [1733885024.141308]: received mask_obs with header seq: 4120 +stamp: + secs: 1733885024 + nsecs: 145607709 +frame_id: '' and grasped_object_mask header seq: 100623 +stamp: + secs: 1733885023 + nsecs: 875909328 +frame_id: "camera_color_optical_frame" +[INFO] [1733885024.334855]: received mask_obs with header seq: 4121 +stamp: + secs: 1733885024 + nsecs: 338650465 +frame_id: '' and grasped_object_mask header seq: 100629 +stamp: + secs: 1733885024 + nsecs: 75750589 +frame_id: "camera_color_optical_frame" +[INFO] [1733885024.518414]: received mask_obs with header seq: 4122 +stamp: + secs: 1733885024 + nsecs: 522666692 +frame_id: '' and grasped_object_mask header seq: 100635 +stamp: + secs: 1733885024 + nsecs: 276950359 +frame_id: "camera_color_optical_frame" +[INFO] [1733885024.713708]: received mask_obs with header seq: 4123 +stamp: + secs: 1733885024 + nsecs: 717934131 +frame_id: '' and grasped_object_mask header seq: 100641 +stamp: + secs: 1733885024 + nsecs: 479137897 +frame_id: "camera_color_optical_frame" +[INFO] [1733885024.915220]: received mask_obs with header seq: 4124 +stamp: + secs: 1733885024 + nsecs: 919136047 +frame_id: '' and grasped_object_mask header seq: 100647 +stamp: + secs: 1733885024 + nsecs: 681411266 +frame_id: "camera_color_optical_frame" +[INFO] [1733885025.108450]: received mask_obs with header seq: 4125 +stamp: + secs: 1733885025 + nsecs: 112655639 +frame_id: '' and grasped_object_mask header seq: 100653 +stamp: + secs: 1733885024 + nsecs: 883771658 +frame_id: "camera_color_optical_frame" +[INFO] [1733885025.309007]: received mask_obs with header seq: 4126 +stamp: + secs: 1733885025 + nsecs: 313205480 +frame_id: '' and grasped_object_mask header seq: 100659 +stamp: + secs: 1733885025 + nsecs: 86219072 +frame_id: "camera_color_optical_frame" +[INFO] [1733885025.508107]: received mask_obs with header seq: 4127 +stamp: + secs: 1733885025 + nsecs: 512339353 +frame_id: '' and grasped_object_mask header seq: 100665 +stamp: + secs: 1733885025 + nsecs: 287013769 +frame_id: "camera_color_optical_frame" +[INFO] [1733885025.721710]: received mask_obs with header seq: 4128 +stamp: + secs: 1733885025 + nsecs: 725837469 +frame_id: '' and grasped_object_mask header seq: 100671 +stamp: + secs: 1733885025 + nsecs: 486630678 +frame_id: "camera_color_optical_frame" +[INFO] [1733885025.916298]: received mask_obs with header seq: 4129 +stamp: + secs: 1733885025 + nsecs: 920453310 +frame_id: '' and grasped_object_mask header seq: 100677 +stamp: + secs: 1733885025 + nsecs: 686237335 +frame_id: "camera_color_optical_frame" +[INFO] [1733885026.114174]: received mask_obs with header seq: 4130 +stamp: + secs: 1733885026 + nsecs: 118306875 +frame_id: '' and grasped_object_mask header seq: 100683 +stamp: + secs: 1733885025 + nsecs: 885831833 +frame_id: "camera_color_optical_frame" +[INFO] [1733885026.324131]: received mask_obs with header seq: 4131 +stamp: + secs: 1733885026 + nsecs: 328327178 +frame_id: '' and grasped_object_mask header seq: 100689 +stamp: + secs: 1733885026 + nsecs: 85414886 +frame_id: "camera_color_optical_frame" +[INFO] [1733885026.524561]: received mask_obs with header seq: 4132 +stamp: + secs: 1733885026 + nsecs: 528685569 +frame_id: '' and grasped_object_mask header seq: 100695 +stamp: + secs: 1733885026 + nsecs: 285013437 +frame_id: "camera_color_optical_frame" +[INFO] [1733885026.712886]: received mask_obs with header seq: 4133 +stamp: + secs: 1733885026 + nsecs: 717162609 +frame_id: '' and grasped_object_mask header seq: 100701 +stamp: + secs: 1733885026 + nsecs: 484618664 +frame_id: "camera_color_optical_frame" +[INFO] [1733885026.908734]: received mask_obs with header seq: 4134 +stamp: + secs: 1733885026 + nsecs: 912924051 +frame_id: '' and grasped_object_mask header seq: 100707 +stamp: + secs: 1733885026 + nsecs: 684212208 +frame_id: "camera_color_optical_frame" +[INFO] [1733885027.114042]: received mask_obs with header seq: 4135 +stamp: + secs: 1733885027 + nsecs: 118155956 +frame_id: '' and grasped_object_mask header seq: 100713 +stamp: + secs: 1733885026 + nsecs: 883794308 +frame_id: "camera_color_optical_frame" +[INFO] [1733885027.315848]: received mask_obs with header seq: 4136 +stamp: + secs: 1733885027 + nsecs: 319969892 +frame_id: '' and grasped_object_mask header seq: 100719 +stamp: + secs: 1733885027 + nsecs: 83365440 +frame_id: "camera_color_optical_frame" +[INFO] [1733885027.523163]: received mask_obs with header seq: 4137 +stamp: + secs: 1733885027 + nsecs: 527378797 +frame_id: '' and grasped_object_mask header seq: 100725 +stamp: + secs: 1733885027 + nsecs: 282959938 +frame_id: "camera_color_optical_frame" +[INFO] [1733885027.714266]: received mask_obs with header seq: 4138 +stamp: + secs: 1733885027 + nsecs: 718507528 +frame_id: '' and grasped_object_mask header seq: 100731 +stamp: + secs: 1733885027 + nsecs: 482567549 +frame_id: "camera_color_optical_frame" +[INFO] [1733885027.918204]: received mask_obs with header seq: 4139 +stamp: + secs: 1733885027 + nsecs: 922418355 +frame_id: '' and grasped_object_mask header seq: 100737 +stamp: + secs: 1733885027 + nsecs: 682163715 +frame_id: "camera_color_optical_frame" +[INFO] [1733885028.109815]: received mask_obs with header seq: 4140 +stamp: + secs: 1733885028 + nsecs: 113697052 +frame_id: '' and grasped_object_mask header seq: 100743 +stamp: + secs: 1733885027 + nsecs: 881748199 +frame_id: "camera_color_optical_frame" +[INFO] [1733885028.318889]: received mask_obs with header seq: 4141 +stamp: + secs: 1733885028 + nsecs: 322912693 +frame_id: '' and grasped_object_mask header seq: 100749 +stamp: + secs: 1733885028 + nsecs: 81321239 +frame_id: "camera_color_optical_frame" +[INFO] [1733885028.521965]: received mask_obs with header seq: 4142 +stamp: + secs: 1733885028 + nsecs: 526143789 +frame_id: '' and grasped_object_mask header seq: 100755 +stamp: + secs: 1733885028 + nsecs: 280915737 +frame_id: "camera_color_optical_frame" +[INFO] [1733885028.720251]: received mask_obs with header seq: 4143 +stamp: + secs: 1733885028 + nsecs: 724446296 +frame_id: '' and grasped_object_mask header seq: 100761 +stamp: + secs: 1733885028 + nsecs: 480520010 +frame_id: "camera_color_optical_frame" +[INFO] [1733885028.919179]: received mask_obs with header seq: 4144 +stamp: + secs: 1733885028 + nsecs: 923285007 +frame_id: '' and grasped_object_mask header seq: 100767 +stamp: + secs: 1733885028 + nsecs: 680112600 +frame_id: "camera_color_optical_frame" +[INFO] [1733885029.126861]: received mask_obs with header seq: 4145 +stamp: + secs: 1733885029 + nsecs: 131077527 +frame_id: '' and grasped_object_mask header seq: 100773 +stamp: + secs: 1733885028 + nsecs: 879693031 +frame_id: "camera_color_optical_frame" +[INFO] [1733885029.310475]: received mask_obs with header seq: 4146 +stamp: + secs: 1733885029 + nsecs: 314053773 +frame_id: '' and grasped_object_mask header seq: 100779 +stamp: + secs: 1733885029 + nsecs: 79261780 +frame_id: "camera_color_optical_frame" +[INFO] [1733885029.507990]: received mask_obs with header seq: 4147 +stamp: + secs: 1733885029 + nsecs: 510348320 +frame_id: '' and grasped_object_mask header seq: 100785 +stamp: + secs: 1733885029 + nsecs: 278852940 +frame_id: "camera_color_optical_frame" +[INFO] [1733885029.715203]: received mask_obs with header seq: 4148 +stamp: + secs: 1733885029 + nsecs: 718974351 +frame_id: '' and grasped_object_mask header seq: 100791 +stamp: + secs: 1733885029 + nsecs: 478457451 +frame_id: "camera_color_optical_frame" +[INFO] [1733885029.914370]: received mask_obs with header seq: 4149 +stamp: + secs: 1733885029 + nsecs: 918393373 +frame_id: '' and grasped_object_mask header seq: 100797 +stamp: + secs: 1733885029 + nsecs: 678049564 +frame_id: "camera_color_optical_frame" +[INFO] [1733885030.117324]: received mask_obs with header seq: 4150 +stamp: + secs: 1733885030 + nsecs: 121278524 +frame_id: '' and grasped_object_mask header seq: 100803 +stamp: + secs: 1733885029 + nsecs: 877629280 +frame_id: "camera_color_optical_frame" +[INFO] [1733885030.312475]: received mask_obs with header seq: 4151 +stamp: + secs: 1733885030 + nsecs: 316589832 +frame_id: '' and grasped_object_mask header seq: 100809 +stamp: + secs: 1733885030 + nsecs: 77197790 +frame_id: "camera_color_optical_frame" +[INFO] [1733885030.524393]: received mask_obs with header seq: 4152 +stamp: + secs: 1733885030 + nsecs: 528617143 +frame_id: '' and grasped_object_mask header seq: 100816 +stamp: + secs: 1733885030 + nsecs: 310059071 +frame_id: "camera_color_optical_frame" +[INFO] [1733885030.727932]: received mask_obs with header seq: 4153 +stamp: + secs: 1733885030 + nsecs: 731332540 +frame_id: '' and grasped_object_mask header seq: 100822 +stamp: + secs: 1733885030 + nsecs: 509663343 +frame_id: "camera_color_optical_frame" +[INFO] [1733885030.934466]: received mask_obs with header seq: 4154 +stamp: + secs: 1733885030 + nsecs: 938694715 +frame_id: '' and grasped_object_mask header seq: 100828 +stamp: + secs: 1733885030 + nsecs: 709256172 +frame_id: "camera_color_optical_frame" +[INFO] [1733885031.142133]: received mask_obs with header seq: 4155 +stamp: + secs: 1733885031 + nsecs: 146311998 +frame_id: '' and grasped_object_mask header seq: 100834 +stamp: + secs: 1733885030 + nsecs: 908837318 +frame_id: "camera_color_optical_frame" +[INFO] [1733885031.353410]: received mask_obs with header seq: 4156 +stamp: + secs: 1733885031 + nsecs: 356190681 +frame_id: '' and grasped_object_mask header seq: 100840 +stamp: + secs: 1733885031 + nsecs: 108407021 +frame_id: "camera_color_optical_frame" +[INFO] [1733885031.545911]: received mask_obs with header seq: 4157 +stamp: + secs: 1733885031 + nsecs: 550198554 +frame_id: '' and grasped_object_mask header seq: 100846 +stamp: + secs: 1733885031 + nsecs: 308011055 +frame_id: "camera_color_optical_frame" +[INFO] [1733885031.741381]: received mask_obs with header seq: 4158 +stamp: + secs: 1733885031 + nsecs: 745633840 +frame_id: '' and grasped_object_mask header seq: 100852 +stamp: + secs: 1733885031 + nsecs: 507618427 +frame_id: "camera_color_optical_frame" +[INFO] [1733885031.943058]: received mask_obs with header seq: 4159 +stamp: + secs: 1733885031 + nsecs: 947231531 +frame_id: '' and grasped_object_mask header seq: 100858 +stamp: + secs: 1733885031 + nsecs: 707213879 +frame_id: "camera_color_optical_frame" +[INFO] [1733885032.138383]: received mask_obs with header seq: 4160 +stamp: + secs: 1733885032 + nsecs: 142337560 +frame_id: '' and grasped_object_mask header seq: 100864 +stamp: + secs: 1733885031 + nsecs: 906798124 +frame_id: "camera_color_optical_frame" +[INFO] [1733885032.328858]: received mask_obs with header seq: 4161 +stamp: + secs: 1733885032 + nsecs: 332664728 +frame_id: '' and grasped_object_mask header seq: 100870 +stamp: + secs: 1733885032 + nsecs: 106370687 +frame_id: "camera_color_optical_frame" +[INFO] [1733885032.533154]: received mask_obs with header seq: 4162 +stamp: + secs: 1733885032 + nsecs: 537288427 +frame_id: '' and grasped_object_mask header seq: 100876 +stamp: + secs: 1733885032 + nsecs: 305969000 +frame_id: "camera_color_optical_frame" +[INFO] [1733885032.733443]: received mask_obs with header seq: 4163 +stamp: + secs: 1733885032 + nsecs: 737694978 +frame_id: '' and grasped_object_mask header seq: 100882 +stamp: + secs: 1733885032 + nsecs: 505569220 +frame_id: "camera_color_optical_frame" +[INFO] [1733885032.923341]: received mask_obs with header seq: 4164 +stamp: + secs: 1733885032 + nsecs: 926932811 +frame_id: '' and grasped_object_mask header seq: 100888 +stamp: + secs: 1733885032 + nsecs: 705156803 +frame_id: "camera_color_optical_frame" +[INFO] [1733885033.118960]: received mask_obs with header seq: 4165 +stamp: + secs: 1733885033 + nsecs: 123193502 +frame_id: '' and grasped_object_mask header seq: 100894 +stamp: + secs: 1733885032 + nsecs: 904732466 +frame_id: "camera_color_optical_frame" +[INFO] [1733885033.321460]: received mask_obs with header seq: 4166 +stamp: + secs: 1733885033 + nsecs: 324962854 +frame_id: '' and grasped_object_mask header seq: 100900 +stamp: + secs: 1733885033 + nsecs: 104295969 +frame_id: "camera_color_optical_frame" +[INFO] [1733885033.518413]: received mask_obs with header seq: 4167 +stamp: + secs: 1733885033 + nsecs: 520938158 +frame_id: '' and grasped_object_mask header seq: 100906 +stamp: + secs: 1733885033 + nsecs: 303898573 +frame_id: "camera_color_optical_frame" +[INFO] [1733885033.716179]: received mask_obs with header seq: 4168 +stamp: + secs: 1733885033 + nsecs: 719160318 +frame_id: '' and grasped_object_mask header seq: 100912 +stamp: + secs: 1733885033 + nsecs: 503507853 +frame_id: "camera_color_optical_frame" +[INFO] [1733885033.920982]: received mask_obs with header seq: 4169 +stamp: + secs: 1733885033 + nsecs: 924358606 +frame_id: '' and grasped_object_mask header seq: 100918 +stamp: + secs: 1733885033 + nsecs: 703106403 +frame_id: "camera_color_optical_frame" +[INFO] [1733885034.124103]: received mask_obs with header seq: 4170 +stamp: + secs: 1733885034 + nsecs: 127185106 +frame_id: '' and grasped_object_mask header seq: 100924 +stamp: + secs: 1733885033 + nsecs: 902692556 +frame_id: "camera_color_optical_frame" +[INFO] [1733885034.317619]: received mask_obs with header seq: 4171 +stamp: + secs: 1733885034 + nsecs: 320910692 +frame_id: '' and grasped_object_mask header seq: 100930 +stamp: + secs: 1733885034 + nsecs: 102267265 +frame_id: "camera_color_optical_frame" +[INFO] [1733885034.528963]: received mask_obs with header seq: 4172 +stamp: + secs: 1733885034 + nsecs: 533242464 +frame_id: '' and grasped_object_mask header seq: 100936 +stamp: + secs: 1733885034 + nsecs: 301873684 +frame_id: "camera_color_optical_frame" +[INFO] [1733885034.726745]: received mask_obs with header seq: 4173 +stamp: + secs: 1733885034 + nsecs: 730997323 +frame_id: '' and grasped_object_mask header seq: 100942 +stamp: + secs: 1733885034 + nsecs: 501484156 +frame_id: "camera_color_optical_frame" +[INFO] [1733885034.929395]: received mask_obs with header seq: 4174 +stamp: + secs: 1733885034 + nsecs: 933667421 +frame_id: '' and grasped_object_mask header seq: 100948 +stamp: + secs: 1733885034 + nsecs: 701083183 +frame_id: "camera_color_optical_frame" +[INFO] [1733885035.121363]: received mask_obs with header seq: 4175 +stamp: + secs: 1733885035 + nsecs: 125288248 +frame_id: '' and grasped_object_mask header seq: 100954 +stamp: + secs: 1733885034 + nsecs: 900671005 +frame_id: "camera_color_optical_frame" +[INFO] [1733885035.325486]: received mask_obs with header seq: 4176 +stamp: + secs: 1733885035 + nsecs: 329768180 +frame_id: '' and grasped_object_mask header seq: 100960 +stamp: + secs: 1733885035 + nsecs: 100248098 +frame_id: "camera_color_optical_frame" +[INFO] [1733885035.519193]: received mask_obs with header seq: 4177 +stamp: + secs: 1733885035 + nsecs: 523139953 +frame_id: '' and grasped_object_mask header seq: 100966 +stamp: + secs: 1733885035 + nsecs: 299849987 +frame_id: "camera_color_optical_frame" +[INFO] [1733885035.722938]: received mask_obs with header seq: 4178 +stamp: + secs: 1733885035 + nsecs: 726904392 +frame_id: '' and grasped_object_mask header seq: 100972 +stamp: + secs: 1733885035 + nsecs: 499454021 +frame_id: "camera_color_optical_frame" +[INFO] [1733885035.925497]: received mask_obs with header seq: 4179 +stamp: + secs: 1733885035 + nsecs: 929741382 +frame_id: '' and grasped_object_mask header seq: 100978 +stamp: + secs: 1733885035 + nsecs: 699046135 +frame_id: "camera_color_optical_frame" +[INFO] [1733885036.115253]: received mask_obs with header seq: 4180 +stamp: + secs: 1733885036 + nsecs: 117075443 +frame_id: '' and grasped_object_mask header seq: 100984 +stamp: + secs: 1733885035 + nsecs: 898626804 +frame_id: "camera_color_optical_frame" +[INFO] [1733885036.316893]: received mask_obs with header seq: 4181 +stamp: + secs: 1733885036 + nsecs: 320775270 +frame_id: '' and grasped_object_mask header seq: 100990 +stamp: + secs: 1733885036 + nsecs: 98195314 +frame_id: "camera_color_optical_frame" +[INFO] [1733885036.518781]: received mask_obs with header seq: 4182 +stamp: + secs: 1733885036 + nsecs: 522797584 +frame_id: '' and grasped_object_mask header seq: 100996 +stamp: + secs: 1733885036 + nsecs: 297803640 +frame_id: "camera_color_optical_frame" +[INFO] [1733885036.726345]: received mask_obs with header seq: 4183 +stamp: + secs: 1733885036 + nsecs: 730477094 +frame_id: '' and grasped_object_mask header seq: 101002 +stamp: + secs: 1733885036 + nsecs: 497417927 +frame_id: "camera_color_optical_frame" +[INFO] [1733885036.920545]: received mask_obs with header seq: 4184 +stamp: + secs: 1733885036 + nsecs: 924442052 +frame_id: '' and grasped_object_mask header seq: 101008 +stamp: + secs: 1733885036 + nsecs: 697021008 +frame_id: "camera_color_optical_frame" +[INFO] [1733885037.117603]: received mask_obs with header seq: 4185 +stamp: + secs: 1733885037 + nsecs: 121859073 +frame_id: '' and grasped_object_mask header seq: 101014 +stamp: + secs: 1733885036 + nsecs: 896612883 +frame_id: "camera_color_optical_frame" +[INFO] [1733885037.321401]: received mask_obs with header seq: 4186 +stamp: + secs: 1733885037 + nsecs: 325635910 +frame_id: '' and grasped_object_mask header seq: 101020 +stamp: + secs: 1733885037 + nsecs: 96193075 +frame_id: "camera_color_optical_frame" +[INFO] [1733885037.517154]: received mask_obs with header seq: 4187 +stamp: + secs: 1733885037 + nsecs: 521326780 +frame_id: '' and grasped_object_mask header seq: 101026 +stamp: + secs: 1733885037 + nsecs: 295793533 +frame_id: "camera_color_optical_frame" +[INFO] [1733885037.720032]: received mask_obs with header seq: 4188 +stamp: + secs: 1733885037 + nsecs: 724153757 +frame_id: '' and grasped_object_mask header seq: 101032 +stamp: + secs: 1733885037 + nsecs: 495394468 +frame_id: "camera_color_optical_frame" +[INFO] [1733885037.928771]: received mask_obs with header seq: 4189 +stamp: + secs: 1733885037 + nsecs: 932899951 +frame_id: '' and grasped_object_mask header seq: 101038 +stamp: + secs: 1733885037 + nsecs: 694982767 +frame_id: "camera_color_optical_frame" +[INFO] [1733885038.130860]: received mask_obs with header seq: 4190 +stamp: + secs: 1733885038 + nsecs: 134989738 +frame_id: '' and grasped_object_mask header seq: 101044 +stamp: + secs: 1733885037 + nsecs: 894558907 +frame_id: "camera_color_optical_frame" +[INFO] [1733885038.329366]: received mask_obs with header seq: 4191 +stamp: + secs: 1733885038 + nsecs: 333353519 +frame_id: '' and grasped_object_mask header seq: 101050 +stamp: + secs: 1733885038 + nsecs: 94122648 +frame_id: "camera_color_optical_frame" +[INFO] [1733885038.528781]: received mask_obs with header seq: 4192 +stamp: + secs: 1733885038 + nsecs: 532637119 +frame_id: '' and grasped_object_mask header seq: 101056 +stamp: + secs: 1733885038 + nsecs: 293729305 +frame_id: "camera_color_optical_frame" +[INFO] [1733885038.723122]: received mask_obs with header seq: 4193 +stamp: + secs: 1733885038 + nsecs: 727354526 +frame_id: '' and grasped_object_mask header seq: 101062 +stamp: + secs: 1733885038 + nsecs: 493344784 +frame_id: "camera_color_optical_frame" +[INFO] [1733885038.918479]: received mask_obs with header seq: 4194 +stamp: + secs: 1733885038 + nsecs: 922558069 +frame_id: '' and grasped_object_mask header seq: 101068 +stamp: + secs: 1733885038 + nsecs: 692950010 +frame_id: "camera_color_optical_frame" +[INFO] [1733885039.116538]: received mask_obs with header seq: 4195 +stamp: + secs: 1733885039 + nsecs: 120812177 +frame_id: '' and grasped_object_mask header seq: 101074 +stamp: + secs: 1733885038 + nsecs: 892543077 +frame_id: "camera_color_optical_frame" +[INFO] [1733885039.318506]: received mask_obs with header seq: 4196 +stamp: + secs: 1733885039 + nsecs: 322695732 +frame_id: '' and grasped_object_mask header seq: 101080 +stamp: + secs: 1733885039 + nsecs: 92124462 +frame_id: "camera_color_optical_frame" +[INFO] [1733885039.513341]: received mask_obs with header seq: 4197 +stamp: + secs: 1733885039 + nsecs: 516619682 +frame_id: '' and grasped_object_mask header seq: 101086 +stamp: + secs: 1733885039 + nsecs: 291728020 +frame_id: "camera_color_optical_frame" +[INFO] [1733885039.716146]: received mask_obs with header seq: 4198 +stamp: + secs: 1733885039 + nsecs: 720349073 +frame_id: '' and grasped_object_mask header seq: 101092 +stamp: + secs: 1733885039 + nsecs: 491336823 +frame_id: "camera_color_optical_frame" +[INFO] [1733885039.915723]: received mask_obs with header seq: 4199 +stamp: + secs: 1733885039 + nsecs: 919960260 +frame_id: '' and grasped_object_mask header seq: 101098 +stamp: + secs: 1733885039 + nsecs: 690933943 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.112323]: received mask_obs with header seq: 4200 +stamp: + secs: 1733885040 + nsecs: 116567373 +frame_id: '' and grasped_object_mask header seq: 101104 +stamp: + secs: 1733885039 + nsecs: 890520334 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.324123]: received mask_obs with header seq: 4201 +stamp: + secs: 1733885040 + nsecs: 328125000 +frame_id: '' and grasped_object_mask header seq: 101110 +stamp: + secs: 1733885040 + nsecs: 90094328 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.514396]: received mask_obs with header seq: 4202 +stamp: + secs: 1733885040 + nsecs: 518410205 +frame_id: '' and grasped_object_mask header seq: 101116 +stamp: + secs: 1733885040 + nsecs: 290539265 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.713584]: received mask_obs with header seq: 4203 +stamp: + secs: 1733885040 + nsecs: 716670513 +frame_id: '' and grasped_object_mask header seq: 101122 +stamp: + secs: 1733885040 + nsecs: 491496801 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.916781]: received mask_obs with header seq: 4204 +stamp: + secs: 1733885040 + nsecs: 920871734 +frame_id: '' and grasped_object_mask header seq: 101128 +stamp: + secs: 1733885040 + nsecs: 692541599 +frame_id: "camera_color_optical_frame" +[INFO] [1733885040.966362]: Storing episode... +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1733885041.115178]: received mask_obs with header seq: 4205 +stamp: + secs: 1733885041 + nsecs: 115984678 +frame_id: '' and grasped_object_mask header seq: 101134 +stamp: + secs: 1733885040 + nsecs: 893673658 +frame_id: "camera_color_optical_frame" +[INFO] [1733885041.740925]: Stored episode 4. +[INFO] [1733885041.743065]: Saving video... +[INFO] [1733885042.254077]: Video saved! +Starting episode 4 +[INFO] [1733885042.294791]: resetting environment +[INFO] [1733885042.317040]: cleared current plan +[INFO] [1733885042.318823]: moving to home +[WARN] [1733885042.304030]: Plan exhausted +[INFO] [1733885045.131358]: reached home +[INFO] [1733885045.144922]: reset action history +[INFO] [1733885047.934548]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733885050.389535]: resetting environment +[INFO] [1733885050.409641]: cleared current plan +[INFO] [1733885050.411672]: moving to home +[INFO] [1733885051.414413]: reached home +[INFO] [1733885051.417237]: reset action history +[INFO] [1733885054.143276]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/episode_rosbags/episode_4_2024-12-10-21-44-14.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-608694.log +started roslaunch server http://158.130.50.37:39083/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [609587] +process[print_text-14]: started with pid [609588] +process[rosbag_record-15]: started with pid [609589] +started bagging! +[INFO] [1733885055.011498]: received mask_obs with header seq: 4206 +stamp: + secs: 1733885055 + nsecs: 14642000 +frame_id: '' and grasped_object_mask header seq: 101551 +stamp: + secs: 1733885054 + nsecs: 794703484 +frame_id: "camera_color_optical_frame" +[INFO] [1733885055.210392]: received mask_obs with header seq: 4207 +stamp: + secs: 1733885055 + nsecs: 213936090 +frame_id: '' and grasped_object_mask header seq: 101558 +stamp: + secs: 1733885055 + nsecs: 27547598 +frame_id: "camera_color_optical_frame" +[INFO] [1733885055.416003]: received mask_obs with header seq: 4208 +stamp: + secs: 1733885055 + nsecs: 420135021 +frame_id: '' and grasped_object_mask header seq: 101563 +stamp: + secs: 1733885055 + nsecs: 193854570 +frame_id: "camera_color_optical_frame" +[INFO] [1733885055.618886]: received mask_obs with header seq: 4209 +stamp: + secs: 1733885055 + nsecs: 623061418 +frame_id: '' and grasped_object_mask header seq: 101569 +stamp: + secs: 1733885055 + nsecs: 393468380 +frame_id: "camera_color_optical_frame" +[INFO] [1733885055.821672]: received mask_obs with header seq: 4210 +stamp: + secs: 1733885055 + nsecs: 825700759 +frame_id: '' and grasped_object_mask header seq: 101576 +stamp: + secs: 1733885055 + nsecs: 626348019 +frame_id: "camera_color_optical_frame" +[INFO] [1733885056.025162]: received mask_obs with header seq: 4211 +stamp: + secs: 1733885056 + nsecs: 28795480 +frame_id: '' and grasped_object_mask header seq: 101582 +stamp: + secs: 1733885055 + nsecs: 825947285 +frame_id: "camera_color_optical_frame" +[INFO] [1733885056.231852]: received mask_obs with header seq: 4212 +stamp: + secs: 1733885056 + nsecs: 234453916 +frame_id: '' and grasped_object_mask header seq: 101588 +stamp: + secs: 1733885056 + nsecs: 25536299 +frame_id: "camera_color_optical_frame" +[INFO] [1733885056.431689]: received mask_obs with header seq: 4213 +stamp: + secs: 1733885056 + nsecs: 434704303 +frame_id: '' and grasped_object_mask header seq: 101594 +stamp: + secs: 1733885056 + nsecs: 225113392 +frame_id: "camera_color_optical_frame" +[INFO] [1733885056.628750]: received mask_obs with header seq: 4214 +stamp: + secs: 1733885056 + nsecs: 633027553 +frame_id: '' and grasped_object_mask header seq: 101600 +stamp: + secs: 1733885056 + nsecs: 424735069 +frame_id: "camera_color_optical_frame" +[INFO] [1733885056.833335]: received mask_obs with header seq: 4215 +stamp: + secs: 1733885056 + nsecs: 837222337 +frame_id: '' and grasped_object_mask header seq: 101606 +stamp: + secs: 1733885056 + nsecs: 624345779 +frame_id: "camera_color_optical_frame" +[INFO] [1733885057.032206]: received mask_obs with header seq: 4216 +stamp: + secs: 1733885057 + nsecs: 35894632 +frame_id: '' and grasped_object_mask header seq: 101612 +stamp: + secs: 1733885056 + nsecs: 823945284 +frame_id: "camera_color_optical_frame" +[INFO] [1733885057.228662]: received mask_obs with header seq: 4217 +stamp: + secs: 1733885057 + nsecs: 232949733 +frame_id: '' and grasped_object_mask header seq: 101618 +stamp: + secs: 1733885057 + nsecs: 23533583 +frame_id: "camera_color_optical_frame" +[INFO] [1733885057.432296]: received mask_obs with header seq: 4218 +stamp: + secs: 1733885057 + nsecs: 436487197 +frame_id: '' and grasped_object_mask header seq: 101624 +stamp: + secs: 1733885057 + nsecs: 223111868 +frame_id: "camera_color_optical_frame" +[INFO] [1733885057.630693]: received mask_obs with header seq: 4219 +stamp: + secs: 1733885057 + nsecs: 634383440 +frame_id: '' and grasped_object_mask header seq: 101630 +stamp: + secs: 1733885057 + nsecs: 422726870 +frame_id: "camera_color_optical_frame" +[INFO] [1733885057.830597]: received mask_obs with header seq: 4220 +stamp: + secs: 1733885057 + nsecs: 833620071 +frame_id: '' and grasped_object_mask header seq: 101636 +stamp: + secs: 1733885057 + nsecs: 622330189 +frame_id: "camera_color_optical_frame" +[INFO] [1733885058.028343]: received mask_obs with header seq: 4221 +stamp: + secs: 1733885058 + nsecs: 32275438 +frame_id: '' and grasped_object_mask header seq: 101642 +stamp: + secs: 1733885057 + nsecs: 821921587 +frame_id: "camera_color_optical_frame" +[INFO] [1733885058.230380]: received mask_obs with header seq: 4222 +stamp: + secs: 1733885058 + nsecs: 233669519 +frame_id: '' and grasped_object_mask header seq: 101648 +stamp: + secs: 1733885058 + nsecs: 21501780 +frame_id: "camera_color_optical_frame" +[INFO] [1733885058.431524]: received mask_obs with header seq: 4223 +stamp: + secs: 1733885058 + nsecs: 435773372 +frame_id: '' and grasped_object_mask header seq: 101654 +stamp: + secs: 1733885058 + nsecs: 221069813 +frame_id: "camera_color_optical_frame" +[INFO] [1733885058.630989]: received mask_obs with header seq: 4224 +stamp: + secs: 1733885058 + nsecs: 633822917 +frame_id: '' and grasped_object_mask header seq: 101660 +stamp: + secs: 1733885058 + nsecs: 420684338 +frame_id: "camera_color_optical_frame" +[INFO] [1733885058.836839]: received mask_obs with header seq: 4225 +stamp: + secs: 1733885058 + nsecs: 840834617 +frame_id: '' and grasped_object_mask header seq: 101666 +stamp: + secs: 1733885058 + nsecs: 620285988 +frame_id: "camera_color_optical_frame" +[INFO] [1733885059.037344]: received mask_obs with header seq: 4226 +stamp: + secs: 1733885059 + nsecs: 40684223 +frame_id: '' and grasped_object_mask header seq: 101672 +stamp: + secs: 1733885058 + nsecs: 819875956 +frame_id: "camera_color_optical_frame" +[INFO] [1733885059.241841]: received mask_obs with header seq: 4227 +stamp: + secs: 1733885059 + nsecs: 245901346 +frame_id: '' and grasped_object_mask header seq: 101678 +stamp: + secs: 1733885059 + nsecs: 19454002 +frame_id: "camera_color_optical_frame" +[INFO] [1733885059.437194]: received mask_obs with header seq: 4228 +stamp: + secs: 1733885059 + nsecs: 440415859 +frame_id: '' and grasped_object_mask header seq: 101684 +stamp: + secs: 1733885059 + nsecs: 219020605 +frame_id: "camera_color_optical_frame" +[INFO] [1733885059.640373]: received mask_obs with header seq: 4229 +stamp: + secs: 1733885059 + nsecs: 643856287 +frame_id: '' and grasped_object_mask header seq: 101690 +stamp: + secs: 1733885059 + nsecs: 418635845 +frame_id: "camera_color_optical_frame" +[INFO] [1733885059.836459]: received mask_obs with header seq: 4230 +stamp: + secs: 1733885059 + nsecs: 839667797 +frame_id: '' and grasped_object_mask header seq: 101696 +stamp: + secs: 1733885059 + nsecs: 618240118 +frame_id: "camera_color_optical_frame" +[INFO] [1733885060.030589]: received mask_obs with header seq: 4231 +stamp: + secs: 1733885060 + nsecs: 32858610 +frame_id: '' and grasped_object_mask header seq: 101702 +stamp: + secs: 1733885059 + nsecs: 817831993 +frame_id: "camera_color_optical_frame" +[INFO] [1733885060.227507]: received mask_obs with header seq: 4232 +stamp: + secs: 1733885060 + nsecs: 230911016 +frame_id: '' and grasped_object_mask header seq: 101708 +stamp: + secs: 1733885060 + nsecs: 17412186 +frame_id: "camera_color_optical_frame" +[INFO] [1733885060.437022]: received mask_obs with header seq: 4233 +stamp: + secs: 1733885060 + nsecs: 441227912 +frame_id: '' and grasped_object_mask header seq: 101714 +stamp: + secs: 1733885060 + nsecs: 216980457 +frame_id: "camera_color_optical_frame" +[INFO] [1733885060.643530]: received mask_obs with header seq: 4234 +stamp: + secs: 1733885060 + nsecs: 647799968 +frame_id: '' and grasped_object_mask header seq: 101720 +stamp: + secs: 1733885060 + nsecs: 416595221 +frame_id: "camera_color_optical_frame" +[INFO] [1733885060.844342]: received mask_obs with header seq: 4235 +stamp: + secs: 1733885060 + nsecs: 848536252 +frame_id: '' and grasped_object_mask header seq: 101726 +stamp: + secs: 1733885060 + nsecs: 616198301 +frame_id: "camera_color_optical_frame" +[INFO] [1733885061.044703]: received mask_obs with header seq: 4236 +stamp: + secs: 1733885061 + nsecs: 48851251 +frame_id: '' and grasped_object_mask header seq: 101732 +stamp: + secs: 1733885060 + nsecs: 815789461 +frame_id: "camera_color_optical_frame" +[INFO] [1733885061.244731]: received mask_obs with header seq: 4237 +stamp: + secs: 1733885061 + nsecs: 248746156 +frame_id: '' and grasped_object_mask header seq: 101738 +stamp: + secs: 1733885061 + nsecs: 15370369 +frame_id: "camera_color_optical_frame" +[INFO] [1733885061.442949]: received mask_obs with header seq: 4238 +stamp: + secs: 1733885061 + nsecs: 447107791 +frame_id: '' and grasped_object_mask header seq: 101744 +stamp: + secs: 1733885061 + nsecs: 214938164 +frame_id: "camera_color_optical_frame" +[INFO] [1733885061.642990]: received mask_obs with header seq: 4239 +stamp: + secs: 1733885061 + nsecs: 647201776 +frame_id: '' and grasped_object_mask header seq: 101750 +stamp: + secs: 1733885061 + nsecs: 414551735 +frame_id: "camera_color_optical_frame" +[INFO] [1733885061.829103]: received mask_obs with header seq: 4240 +stamp: + secs: 1733885061 + nsecs: 832829236 +frame_id: '' and grasped_object_mask header seq: 101756 +stamp: + secs: 1733885061 + nsecs: 614153624 +frame_id: "camera_color_optical_frame" +[INFO] [1733885062.027764]: received mask_obs with header seq: 4241 +stamp: + secs: 1733885062 + nsecs: 30900716 +frame_id: '' and grasped_object_mask header seq: 101762 +stamp: + secs: 1733885061 + nsecs: 813743353 +frame_id: "camera_color_optical_frame" +[INFO] [1733885062.239140]: received mask_obs with header seq: 4242 +stamp: + secs: 1733885062 + nsecs: 243433952 +frame_id: '' and grasped_object_mask header seq: 101768 +stamp: + secs: 1733885062 + nsecs: 13321877 +frame_id: "camera_color_optical_frame" +[INFO] [1733885062.439327]: received mask_obs with header seq: 4243 +stamp: + secs: 1733885062 + nsecs: 443422317 +frame_id: '' and grasped_object_mask header seq: 101774 +stamp: + secs: 1733885062 + nsecs: 212889194 +frame_id: "camera_color_optical_frame" +[INFO] [1733885062.638753]: received mask_obs with header seq: 4244 +stamp: + secs: 1733885062 + nsecs: 642963647 +frame_id: '' and grasped_object_mask header seq: 101780 +stamp: + secs: 1733885062 + nsecs: 412501812 +frame_id: "camera_color_optical_frame" +[INFO] [1733885062.838940]: received mask_obs with header seq: 4245 +stamp: + secs: 1733885062 + nsecs: 843137264 +frame_id: '' and grasped_object_mask header seq: 101786 +stamp: + secs: 1733885062 + nsecs: 612102509 +frame_id: "camera_color_optical_frame" +[INFO] [1733885063.041947]: received mask_obs with header seq: 4246 +stamp: + secs: 1733885063 + nsecs: 45998573 +frame_id: '' and grasped_object_mask header seq: 101792 +stamp: + secs: 1733885062 + nsecs: 811691523 +frame_id: "camera_color_optical_frame" +[INFO] [1733885063.250874]: received mask_obs with header seq: 4247 +stamp: + secs: 1733885063 + nsecs: 254637718 +frame_id: '' and grasped_object_mask header seq: 101798 +stamp: + secs: 1733885063 + nsecs: 11268377 +frame_id: "camera_color_optical_frame" +[INFO] [1733885063.441231]: received mask_obs with header seq: 4248 +stamp: + secs: 1733885063 + nsecs: 445280075 +frame_id: '' and grasped_object_mask header seq: 101804 +stamp: + secs: 1733885063 + nsecs: 210833549 +frame_id: "camera_color_optical_frame" +[INFO] [1733885063.633446]: received mask_obs with header seq: 4249 +stamp: + secs: 1733885063 + nsecs: 637672424 +frame_id: '' and grasped_object_mask header seq: 101810 +stamp: + secs: 1733885063 + nsecs: 410445452 +frame_id: "camera_color_optical_frame" +[INFO] [1733885063.837421]: received mask_obs with header seq: 4250 +stamp: + secs: 1733885063 + nsecs: 841694355 +frame_id: '' and grasped_object_mask header seq: 101816 +stamp: + secs: 1733885063 + nsecs: 610044479 +frame_id: "camera_color_optical_frame" +[INFO] [1733885064.028882]: received mask_obs with header seq: 4251 +stamp: + secs: 1733885064 + nsecs: 33169507 +frame_id: '' and grasped_object_mask header seq: 101822 +stamp: + secs: 1733885063 + nsecs: 809631586 +frame_id: "camera_color_optical_frame" +[INFO] [1733885064.227249]: received mask_obs with header seq: 4252 +stamp: + secs: 1733885064 + nsecs: 231336355 +frame_id: '' and grasped_object_mask header seq: 101828 +stamp: + secs: 1733885064 + nsecs: 9206772 +frame_id: "camera_color_optical_frame" +[INFO] [1733885064.434969]: received mask_obs with header seq: 4253 +stamp: + secs: 1733885064 + nsecs: 439094305 +frame_id: '' and grasped_object_mask header seq: 101834 +stamp: + secs: 1733885064 + nsecs: 208770037 +frame_id: "camera_color_optical_frame" +[INFO] [1733885064.626339]: received mask_obs with header seq: 4254 +stamp: + secs: 1733885064 + nsecs: 629540681 +frame_id: '' and grasped_object_mask header seq: 101840 +stamp: + secs: 1733885064 + nsecs: 408391714 +frame_id: "camera_color_optical_frame" +[INFO] [1733885064.829620]: received mask_obs with header seq: 4255 +stamp: + secs: 1733885064 + nsecs: 833413839 +frame_id: '' and grasped_object_mask header seq: 101846 +stamp: + secs: 1733885064 + nsecs: 608002663 +frame_id: "camera_color_optical_frame" +[INFO] [1733885065.032439]: received mask_obs with header seq: 4256 +stamp: + secs: 1733885065 + nsecs: 36078691 +frame_id: '' and grasped_object_mask header seq: 101852 +stamp: + secs: 1733885064 + nsecs: 807601452 +frame_id: "camera_color_optical_frame" +[INFO] [1733885065.231527]: received mask_obs with header seq: 4257 +stamp: + secs: 1733885065 + nsecs: 235765695 +frame_id: '' and grasped_object_mask header seq: 101858 +stamp: + secs: 1733885065 + nsecs: 7189274 +frame_id: "camera_color_optical_frame" +[INFO] [1733885065.438609]: received mask_obs with header seq: 4258 +stamp: + secs: 1733885065 + nsecs: 442704200 +frame_id: '' and grasped_object_mask header seq: 101864 +stamp: + secs: 1733885065 + nsecs: 206765652 +frame_id: "camera_color_optical_frame" +[INFO] [1733885065.630610]: received mask_obs with header seq: 4259 +stamp: + secs: 1733885065 + nsecs: 634860754 +frame_id: '' and grasped_object_mask header seq: 101870 +stamp: + secs: 1733885065 + nsecs: 406378984 +frame_id: "camera_color_optical_frame" +[INFO] [1733885065.838525]: received mask_obs with header seq: 4260 +stamp: + secs: 1733885065 + nsecs: 842761516 +frame_id: '' and grasped_object_mask header seq: 101876 +stamp: + secs: 1733885065 + nsecs: 605980396 +frame_id: "camera_color_optical_frame" +[INFO] [1733885066.039594]: received mask_obs with header seq: 4261 +stamp: + secs: 1733885066 + nsecs: 43303251 +frame_id: '' and grasped_object_mask header seq: 101882 +stamp: + secs: 1733885065 + nsecs: 805571079 +frame_id: "camera_color_optical_frame" +[INFO] [1733885066.225931]: received mask_obs with header seq: 4262 +stamp: + secs: 1733885066 + nsecs: 230207443 +frame_id: '' and grasped_object_mask header seq: 101888 +stamp: + secs: 1733885066 + nsecs: 5148888 +frame_id: "camera_color_optical_frame" +[INFO] [1733885066.430456]: received mask_obs with header seq: 4263 +stamp: + secs: 1733885066 + nsecs: 434661388 +frame_id: '' and grasped_object_mask header seq: 101894 +stamp: + secs: 1733885066 + nsecs: 204714537 +frame_id: "camera_color_optical_frame" +[INFO] [1733885066.628259]: received mask_obs with header seq: 4264 +stamp: + secs: 1733885066 + nsecs: 632501363 +frame_id: '' and grasped_object_mask header seq: 101900 +stamp: + secs: 1733885066 + nsecs: 405868292 +frame_id: "camera_color_optical_frame" +[INFO] [1733885066.839576]: received mask_obs with header seq: 4265 +stamp: + secs: 1733885066 + nsecs: 843599319 +frame_id: '' and grasped_object_mask header seq: 101906 +stamp: + secs: 1733885066 + nsecs: 607826471 +frame_id: "camera_color_optical_frame" +[INFO] [1733885067.027728]: received mask_obs with header seq: 4266 +stamp: + secs: 1733885067 + nsecs: 31682968 +frame_id: '' and grasped_object_mask header seq: 101912 +stamp: + secs: 1733885066 + nsecs: 809859276 +frame_id: "camera_color_optical_frame" +[INFO] [1733885067.237664]: received mask_obs with header seq: 4267 +stamp: + secs: 1733885067 + nsecs: 241655349 +frame_id: '' and grasped_object_mask header seq: 101918 +stamp: + secs: 1733885067 + nsecs: 11968613 +frame_id: "camera_color_optical_frame" +[INFO] [1733885067.430653]: received mask_obs with header seq: 4268 +stamp: + secs: 1733885067 + nsecs: 434848785 +frame_id: '' and grasped_object_mask header seq: 101924 +stamp: + secs: 1733885067 + nsecs: 214151621 +frame_id: "camera_color_optical_frame" +[INFO] [1733885067.630760]: received mask_obs with header seq: 4269 +stamp: + secs: 1733885067 + nsecs: 634780406 +frame_id: '' and grasped_object_mask header seq: 101930 +stamp: + secs: 1733885067 + nsecs: 414499760 +frame_id: "camera_color_optical_frame" +[INFO] [1733885067.829261]: received mask_obs with header seq: 4270 +stamp: + secs: 1733885067 + nsecs: 833310604 +frame_id: '' and grasped_object_mask header seq: 101936 +stamp: + secs: 1733885067 + nsecs: 613886356 +frame_id: "camera_color_optical_frame" +[INFO] [1733885068.023604]: received mask_obs with header seq: 4271 +stamp: + secs: 1733885068 + nsecs: 27897596 +frame_id: '' and grasped_object_mask header seq: 101942 +stamp: + secs: 1733885067 + nsecs: 813250303 +frame_id: "camera_color_optical_frame" +[INFO] [1733885068.235771]: received mask_obs with header seq: 4272 +stamp: + secs: 1733885068 + nsecs: 239995718 +frame_id: '' and grasped_object_mask header seq: 101948 +stamp: + secs: 1733885068 + nsecs: 12591124 +frame_id: "camera_color_optical_frame" +[INFO] [1733885068.435628]: received mask_obs with header seq: 4273 +stamp: + secs: 1733885068 + nsecs: 439838171 +frame_id: '' and grasped_object_mask header seq: 101954 +stamp: + secs: 1733885068 + nsecs: 211908817 +frame_id: "camera_color_optical_frame" +[INFO] [1733885068.632806]: received mask_obs with header seq: 4274 +stamp: + secs: 1733885068 + nsecs: 636551856 +frame_id: '' and grasped_object_mask header seq: 101960 +stamp: + secs: 1733885068 + nsecs: 412127972 +frame_id: "camera_color_optical_frame" +[INFO] [1733885068.837899]: received mask_obs with header seq: 4275 +stamp: + secs: 1733885068 + nsecs: 842099905 +frame_id: '' and grasped_object_mask header seq: 101966 +stamp: + secs: 1733885068 + nsecs: 612867117 +frame_id: "camera_color_optical_frame" +[INFO] [1733885069.037648]: received mask_obs with header seq: 4276 +stamp: + secs: 1733885069 + nsecs: 41481733 +frame_id: '' and grasped_object_mask header seq: 101972 +stamp: + secs: 1733885068 + nsecs: 813682556 +frame_id: "camera_color_optical_frame" +[INFO] [1733885069.235460]: received mask_obs with header seq: 4277 +stamp: + secs: 1733885069 + nsecs: 239587306 +frame_id: '' and grasped_object_mask header seq: 101978 +stamp: + secs: 1733885069 + nsecs: 14574528 +frame_id: "camera_color_optical_frame" +[INFO] [1733885069.437453]: received mask_obs with header seq: 4278 +stamp: + secs: 1733885069 + nsecs: 441674470 +frame_id: '' and grasped_object_mask header seq: 101984 +stamp: + secs: 1733885069 + nsecs: 215542555 +frame_id: "camera_color_optical_frame" +[INFO] [1733885069.638594]: received mask_obs with header seq: 4279 +stamp: + secs: 1733885069 + nsecs: 642611980 +frame_id: '' and grasped_object_mask header seq: 101990 +stamp: + secs: 1733885069 + nsecs: 415625811 +frame_id: "camera_color_optical_frame" +[INFO] [1733885069.834238]: received mask_obs with header seq: 4280 +stamp: + secs: 1733885069 + nsecs: 838496923 +frame_id: '' and grasped_object_mask header seq: 101996 +stamp: + secs: 1733885069 + nsecs: 615232706 +frame_id: "camera_color_optical_frame" +[INFO] [1733885070.030226]: received mask_obs with header seq: 4281 +stamp: + secs: 1733885070 + nsecs: 33568859 +frame_id: '' and grasped_object_mask header seq: 102002 +stamp: + secs: 1733885069 + nsecs: 814826965 +frame_id: "camera_color_optical_frame" +[INFO] [1733885070.227584]: received mask_obs with header seq: 4282 +stamp: + secs: 1733885070 + nsecs: 230847835 +frame_id: '' and grasped_object_mask header seq: 102008 +stamp: + secs: 1733885070 + nsecs: 14409542 +frame_id: "camera_color_optical_frame" +[INFO] [1733885070.434891]: received mask_obs with header seq: 4283 +stamp: + secs: 1733885070 + nsecs: 439213037 +frame_id: '' and grasped_object_mask header seq: 102014 +stamp: + secs: 1733885070 + nsecs: 213980436 +frame_id: "camera_color_optical_frame" +[INFO] [1733885070.626748]: received mask_obs with header seq: 4284 +stamp: + secs: 1733885070 + nsecs: 630806684 +frame_id: '' and grasped_object_mask header seq: 102020 +stamp: + secs: 1733885070 + nsecs: 413573980 +frame_id: "camera_color_optical_frame" +[INFO] [1733885070.826396]: received mask_obs with header seq: 4285 +stamp: + secs: 1733885070 + nsecs: 830528497 +frame_id: '' and grasped_object_mask header seq: 102026 +stamp: + secs: 1733885070 + nsecs: 613174200 +frame_id: "camera_color_optical_frame" +[INFO] [1733885071.022197]: received mask_obs with header seq: 4286 +stamp: + secs: 1733885071 + nsecs: 26180505 +frame_id: '' and grasped_object_mask header seq: 102032 +stamp: + secs: 1733885070 + nsecs: 812763691 +frame_id: "camera_color_optical_frame" +[INFO] [1733885071.222132]: received mask_obs with header seq: 4287 +stamp: + secs: 1733885071 + nsecs: 225623846 +frame_id: '' and grasped_object_mask header seq: 102038 +stamp: + secs: 1733885071 + nsecs: 12340069 +frame_id: "camera_color_optical_frame" +[INFO] [1733885071.425158]: received mask_obs with header seq: 4288 +stamp: + secs: 1733885071 + nsecs: 429080486 +frame_id: '' and grasped_object_mask header seq: 102044 +stamp: + secs: 1733885071 + nsecs: 211904526 +frame_id: "camera_color_optical_frame" +[INFO] [1733885071.621522]: received mask_obs with header seq: 4289 +stamp: + secs: 1733885071 + nsecs: 625704765 +frame_id: '' and grasped_object_mask header seq: 102050 +stamp: + secs: 1733885071 + nsecs: 411497355 +frame_id: "camera_color_optical_frame" +[INFO] [1733885071.829549]: received mask_obs with header seq: 4290 +stamp: + secs: 1733885071 + nsecs: 833496332 +frame_id: '' and grasped_object_mask header seq: 102056 +stamp: + secs: 1733885071 + nsecs: 611101627 +frame_id: "camera_color_optical_frame" +[INFO] [1733885072.021947]: received mask_obs with header seq: 4291 +stamp: + secs: 1733885072 + nsecs: 24584531 +frame_id: '' and grasped_object_mask header seq: 102062 +stamp: + secs: 1733885071 + nsecs: 810693979 +frame_id: "camera_color_optical_frame" +[INFO] [1733885072.221440]: received mask_obs with header seq: 4292 +stamp: + secs: 1733885072 + nsecs: 225684404 +frame_id: '' and grasped_object_mask header seq: 102068 +stamp: + secs: 1733885072 + nsecs: 10275841 +frame_id: "camera_color_optical_frame" +[INFO] [1733885072.447968]: received mask_obs with header seq: 4293 +stamp: + secs: 1733885072 + nsecs: 451795816 +frame_id: '' and grasped_object_mask header seq: 102074 +stamp: + secs: 1733885072 + nsecs: 209845304 +frame_id: "camera_color_optical_frame" +[INFO] [1733885072.648247]: received mask_obs with header seq: 4294 +stamp: + secs: 1733885072 + nsecs: 652412652 +frame_id: '' and grasped_object_mask header seq: 102080 +stamp: + secs: 1733885072 + nsecs: 409443140 +frame_id: "camera_color_optical_frame" +[INFO] [1733885072.851180]: received mask_obs with header seq: 4295 +stamp: + secs: 1733885072 + nsecs: 855420827 +frame_id: '' and grasped_object_mask header seq: 102086 +stamp: + secs: 1733885072 + nsecs: 609052658 +frame_id: "camera_color_optical_frame" +[INFO] [1733885073.032058]: received mask_obs with header seq: 4296 +stamp: + secs: 1733885073 + nsecs: 36090612 +frame_id: '' and grasped_object_mask header seq: 102092 +stamp: + secs: 1733885072 + nsecs: 808650970 +frame_id: "camera_color_optical_frame" +[INFO] [1733885073.232475]: received mask_obs with header seq: 4297 +stamp: + secs: 1733885073 + nsecs: 236685276 +frame_id: '' and grasped_object_mask header seq: 102098 +stamp: + secs: 1733885073 + nsecs: 8237839 +frame_id: "camera_color_optical_frame" +[INFO] [1733885073.437500]: received mask_obs with header seq: 4298 +stamp: + secs: 1733885073 + nsecs: 441420555 +frame_id: '' and grasped_object_mask header seq: 102104 +stamp: + secs: 1733885073 + nsecs: 207813978 +frame_id: "camera_color_optical_frame" +[INFO] [1733885073.627487]: received mask_obs with header seq: 4299 +stamp: + secs: 1733885073 + nsecs: 631722688 +frame_id: '' and grasped_object_mask header seq: 102110 +stamp: + secs: 1733885073 + nsecs: 407410145 +frame_id: "camera_color_optical_frame" +[INFO] [1733885073.823214]: received mask_obs with header seq: 4300 +stamp: + secs: 1733885073 + nsecs: 827508687 +frame_id: '' and grasped_object_mask header seq: 102116 +stamp: + secs: 1733885073 + nsecs: 607013464 +frame_id: "camera_color_optical_frame" +[INFO] [1733885074.026474]: received mask_obs with header seq: 4301 +stamp: + secs: 1733885074 + nsecs: 29206037 +frame_id: '' and grasped_object_mask header seq: 102122 +stamp: + secs: 1733885073 + nsecs: 806604862 +frame_id: "camera_color_optical_frame" +[INFO] [1733885074.222903]: received mask_obs with header seq: 4302 +stamp: + secs: 1733885074 + nsecs: 227107763 +frame_id: '' and grasped_object_mask header seq: 102128 +stamp: + secs: 1733885074 + nsecs: 6184816 +frame_id: "camera_color_optical_frame" +[INFO] [1733885074.453949]: received mask_obs with header seq: 4303 +stamp: + secs: 1733885074 + nsecs: 458142042 +frame_id: '' and grasped_object_mask header seq: 102134 +stamp: + secs: 1733885074 + nsecs: 205752611 +frame_id: "camera_color_optical_frame" +[INFO] [1733885074.662881]: received mask_obs with header seq: 4304 +stamp: + secs: 1733885074 + nsecs: 666843175 +frame_id: '' and grasped_object_mask header seq: 102141 +stamp: + secs: 1733885074 + nsecs: 438617468 +frame_id: "camera_color_optical_frame" +[INFO] [1733885074.845430]: received mask_obs with header seq: 4305 +stamp: + secs: 1733885074 + nsecs: 849362611 +frame_id: '' and grasped_object_mask header seq: 102146 +stamp: + secs: 1733885074 + nsecs: 604956150 +frame_id: "camera_color_optical_frame" +[INFO] [1733885075.041323]: received mask_obs with header seq: 4306 +stamp: + secs: 1733885075 + nsecs: 45567274 +frame_id: '' and grasped_object_mask header seq: 102152 +stamp: + secs: 1733885074 + nsecs: 804551125 +frame_id: "camera_color_optical_frame" +[INFO] [1733885075.239355]: received mask_obs with header seq: 4307 +stamp: + secs: 1733885075 + nsecs: 243563890 +frame_id: '' and grasped_object_mask header seq: 102158 +stamp: + secs: 1733885075 + nsecs: 4134417 +frame_id: "camera_color_optical_frame" +[INFO] [1733885075.435836]: received mask_obs with header seq: 4308 +stamp: + secs: 1733885075 + nsecs: 440087318 +frame_id: '' and grasped_object_mask header seq: 102164 +stamp: + secs: 1733885075 + nsecs: 203706264 +frame_id: "camera_color_optical_frame" +[INFO] [1733885075.626316]: received mask_obs with header seq: 4309 +stamp: + secs: 1733885075 + nsecs: 630609035 +frame_id: '' and grasped_object_mask header seq: 102170 +stamp: + secs: 1733885075 + nsecs: 403305292 +frame_id: "camera_color_optical_frame" +[INFO] [1733885075.836604]: received mask_obs with header seq: 4310 +stamp: + secs: 1733885075 + nsecs: 840879917 +frame_id: '' and grasped_object_mask header seq: 102176 +stamp: + secs: 1733885075 + nsecs: 602915287 +frame_id: "camera_color_optical_frame" +[INFO] [1733885076.046784]: received mask_obs with header seq: 4311 +stamp: + secs: 1733885076 + nsecs: 51002264 +frame_id: '' and grasped_object_mask header seq: 102182 +stamp: + secs: 1733885075 + nsecs: 802515030 +frame_id: "camera_color_optical_frame" +[INFO] [1733885076.242701]: received mask_obs with header seq: 4312 +stamp: + secs: 1733885076 + nsecs: 246860027 +frame_id: '' and grasped_object_mask header seq: 102188 +stamp: + secs: 1733885076 + nsecs: 2102375 +frame_id: "camera_color_optical_frame" +[INFO] [1733885076.457050]: received mask_obs with header seq: 4313 +stamp: + secs: 1733885076 + nsecs: 460905075 +frame_id: '' and grasped_object_mask header seq: 102194 +stamp: + secs: 1733885076 + nsecs: 201678038 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 100987760 for nearest: 1733885076502261487 - target: 1733885076401273727 at idx 249 +[INFO] [1733885076.643283]: received mask_obs with header seq: 4314 +stamp: + secs: 1733885076 + nsecs: 647121667 +frame_id: '' and grasped_object_mask header seq: 102200 +stamp: + secs: 1733885076 + nsecs: 401273727 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 101428063 for nearest: 1733885076702304871 - target: 1733885076600876808 at idx 249 +[INFO] [1733885076.853699]: received mask_obs with header seq: 4315 +stamp: + secs: 1733885076 + nsecs: 856991767 +frame_id: '' and grasped_object_mask header seq: 102206 +stamp: + secs: 1733885076 + nsecs: 600876808 +frame_id: "camera_color_optical_frame" +[INFO] [1733885077.057890]: received mask_obs with header seq: 4316 +stamp: + secs: 1733885077 + nsecs: 61761140 +frame_id: '' and grasped_object_mask header seq: 102212 +stamp: + secs: 1733885076 + nsecs: 800467730 +frame_id: "camera_color_optical_frame" +[INFO] [1733885077.270113]: received mask_obs with header seq: 4317 +stamp: + secs: 1733885077 + nsecs: 274136781 +frame_id: '' and grasped_object_mask header seq: 102219 +stamp: + secs: 1733885077 + nsecs: 33309460 +frame_id: "camera_color_optical_frame" +[INFO] [1733885077.465693]: received mask_obs with header seq: 4318 +stamp: + secs: 1733885077 + nsecs: 469737768 +frame_id: '' and grasped_object_mask header seq: 102225 +stamp: + secs: 1733885077 + nsecs: 232875347 +frame_id: "camera_color_optical_frame" +[INFO] [1733885077.659247]: received mask_obs with header seq: 4319 +stamp: + secs: 1733885077 + nsecs: 663504362 +frame_id: '' and grasped_object_mask header seq: 102231 +stamp: + secs: 1733885077 + nsecs: 432476759 +frame_id: "camera_color_optical_frame" +[INFO] [1733885077.878704]: received mask_obs with header seq: 4320 +stamp: + secs: 1733885077 + nsecs: 882984399 +frame_id: '' and grasped_object_mask header seq: 102237 +stamp: + secs: 1733885077 + nsecs: 632080317 +frame_id: "camera_color_optical_frame" +[INFO] [1733885078.059101]: received mask_obs with header seq: 4321 +stamp: + secs: 1733885078 + nsecs: 63217878 +frame_id: '' and grasped_object_mask header seq: 102243 +stamp: + secs: 1733885077 + nsecs: 831671000 +frame_id: "camera_color_optical_frame" +[INFO] [1733885078.252701]: received mask_obs with header seq: 4322 +stamp: + secs: 1733885078 + nsecs: 256977319 +frame_id: '' and grasped_object_mask header seq: 102248 +stamp: + secs: 1733885077 + nsecs: 997987032 +frame_id: "camera_color_optical_frame" +[INFO] [1733885078.456300]: received mask_obs with header seq: 4323 +stamp: + secs: 1733885078 + nsecs: 460353851 +frame_id: '' and grasped_object_mask header seq: 102255 +stamp: + secs: 1733885078 + nsecs: 230816841 +frame_id: "camera_color_optical_frame" +[INFO] [1733885078.664910]: received mask_obs with header seq: 4324 +stamp: + secs: 1733885078 + nsecs: 668450117 +frame_id: '' and grasped_object_mask header seq: 102261 +stamp: + secs: 1733885078 + nsecs: 430422783 +frame_id: "camera_color_optical_frame" +[INFO] [1733885078.869142]: received mask_obs with header seq: 4325 +stamp: + secs: 1733885078 + nsecs: 873075246 +frame_id: '' and grasped_object_mask header seq: 102267 +stamp: + secs: 1733885078 + nsecs: 630031109 +frame_id: "camera_color_optical_frame" +[INFO] [1733885079.058522]: received mask_obs with header seq: 4326 +stamp: + secs: 1733885079 + nsecs: 62856912 +frame_id: '' and grasped_object_mask header seq: 102273 +stamp: + secs: 1733885078 + nsecs: 829628468 +frame_id: "camera_color_optical_frame" +[INFO] [1733885079.261057]: received mask_obs with header seq: 4327 +stamp: + secs: 1733885079 + nsecs: 264467000 +frame_id: '' and grasped_object_mask header seq: 102279 +stamp: + secs: 1733885079 + nsecs: 29213428 +frame_id: "camera_color_optical_frame" +[INFO] [1733885079.462280]: received mask_obs with header seq: 4328 +stamp: + secs: 1733885079 + nsecs: 465749740 +frame_id: '' and grasped_object_mask header seq: 102285 +stamp: + secs: 1733885079 + nsecs: 228786707 +frame_id: "camera_color_optical_frame" +[INFO] [1733885079.668767]: received mask_obs with header seq: 4329 +stamp: + secs: 1733885079 + nsecs: 672876119 +frame_id: '' and grasped_object_mask header seq: 102291 +stamp: + secs: 1733885079 + nsecs: 428391218 +frame_id: "camera_color_optical_frame" +[INFO] [1733885079.855755]: received mask_obs with header seq: 4330 +stamp: + secs: 1733885079 + nsecs: 860065698 +frame_id: '' and grasped_object_mask header seq: 102297 +stamp: + secs: 1733885079 + nsecs: 627995491 +frame_id: "camera_color_optical_frame" +[INFO] [1733885080.056728]: received mask_obs with header seq: 4331 +stamp: + secs: 1733885080 + nsecs: 60037612 +frame_id: '' and grasped_object_mask header seq: 102303 +stamp: + secs: 1733885079 + nsecs: 827588320 +frame_id: "camera_color_optical_frame" +[INFO] [1733885080.259524]: received mask_obs with header seq: 4332 +stamp: + secs: 1733885080 + nsecs: 263851881 +frame_id: '' and grasped_object_mask header seq: 102309 +stamp: + secs: 1733885080 + nsecs: 27170420 +frame_id: "camera_color_optical_frame" +[INFO] [1733885080.456278]: received mask_obs with header seq: 4333 +stamp: + secs: 1733885080 + nsecs: 460145235 +frame_id: '' and grasped_object_mask header seq: 102314 +stamp: + secs: 1733885080 + nsecs: 193478584 +frame_id: "camera_color_optical_frame" +[INFO] [1733885080.657196]: received mask_obs with header seq: 4334 +stamp: + secs: 1733885080 + nsecs: 660703897 +frame_id: '' and grasped_object_mask header seq: 102320 +stamp: + secs: 1733885080 + nsecs: 393074036 +frame_id: "camera_color_optical_frame" +[INFO] [1733885080.856349]: received mask_obs with header seq: 4335 +stamp: + secs: 1733885080 + nsecs: 860479831 +frame_id: '' and grasped_object_mask header seq: 102327 +stamp: + secs: 1733885080 + nsecs: 625947237 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.049817]: received mask_obs with header seq: 4336 +stamp: + secs: 1733885081 + nsecs: 53580999 +frame_id: '' and grasped_object_mask header seq: 102333 +stamp: + secs: 1733885080 + nsecs: 825540066 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.258859]: received mask_obs with header seq: 4337 +stamp: + secs: 1733885081 + nsecs: 263132333 +frame_id: '' and grasped_object_mask header seq: 102339 +stamp: + secs: 1733885081 + nsecs: 25121212 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.444914]: received mask_obs with header seq: 4338 +stamp: + secs: 1733885081 + nsecs: 449147701 +frame_id: '' and grasped_object_mask header seq: 102344 +stamp: + secs: 1733885081 + nsecs: 191430092 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.634412]: received mask_obs with header seq: 4339 +stamp: + secs: 1733885081 + nsecs: 638667821 +frame_id: '' and grasped_object_mask header seq: 102350 +stamp: + secs: 1733885081 + nsecs: 391024351 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.832880]: received mask_obs with header seq: 4340 +stamp: + secs: 1733885081 + nsecs: 837072610 +frame_id: '' and grasped_object_mask header seq: 102356 +stamp: + secs: 1733885081 + nsecs: 590628624 +frame_id: "camera_color_optical_frame" +[INFO] [1733885081.855714]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1733885082.661566]: Stored episode 5. +[INFO] [1733885082.665876]: Saving video... +[INFO] [1733885083.200730]: Video saved! +[INFO] [1733885083.252294]: resetting environment +[INFO] [1733885083.274524]: cleared current plan +[INFO] [1733885083.275739]: moving to home +[WARN] [1733885083.024168]: Plan exhausted +[WARN] [1733885083.064397]: Plan exhausted +[WARN] [1733885083.105489]: Plan exhausted +[WARN] [1733885083.150001]: Plan exhausted +[WARN] [1733885083.185095]: Plan exhausted +[WARN] [1733885083.224035]: Plan exhausted +[WARN] [1733885083.264048]: Plan exhausted +[INFO] [1733885086.084415]: reached home +[INFO] [1733885086.090099]: reset action history +[INFO] [1733885088.888587]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt b/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json b/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fe2e59b6db3ab8fa35595b0f87d6d72e080c3887 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T02:40:38.805785", + "startedAt": "2024-12-11T02:40:38.443244", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3754.252, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4503.722, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 402.6240234375 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json b/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1ba318eafe7d5394f32d33e61814d38a37dbefd4 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "99cde89795fdf8f2318ab5a5df6c7470a8c3f27c70506325712d1f1ae63acf0c", "size": 1153889, "path": "media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4"}, "global_step": 679, "_timestamp": 1733885090.9171271, "_runtime": 252.4643030166626, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "0f1a83ed932af73a96be91a936459619378d8947c1ececae957ffdb603897874", "size": 1157518, "path": "media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/2_eval": {"_type": "video-file", "sha256": "aac838f24394620216df297962eea9126f794122e4334824543af281f264ae29", "size": 1152465, "path": "media/videos/eval/2_eval_2_aac838f24394620216df.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9f3ddf93c395db4f718a9dd95be79aaf7a24dbe1b0621ffe2e060dbef9f228a1", "size": 1212773, "path": "media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "86ec182da7a62ff7b578325c5e5e3f28bb6d9f7cc3d236c696083a8630fa2ae1", "size": 1194589, "path": "media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4"}, "_wandb": {"runtime": 252}} \ No newline at end of file diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug-internal.log b/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2794ac313f302d2e118d21995b461bc2b5b89a68 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug-internal.log @@ -0,0 +1,749 @@ +2024-12-10 21:40:38,452 INFO StreamThr :608881 [internal.py:wandb_internal():85] W&B internal server running at pid: 608881, started at: 2024-12-10 21:40:38.452460 +2024-12-10 21:40:38,453 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:40:38,464 INFO WriterThread:608881 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb +2024-12-10 21:40:38,465 DEBUG SenderThread:608881 [sender.py:send():379] send: header +2024-12-10 21:40:38,473 DEBUG SenderThread:608881 [sender.py:send():379] send: run +2024-12-10 21:40:38,722 INFO SenderThread:608881 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files +2024-12-10 21:40:38,722 INFO SenderThread:608881 [sender.py:_start_run_threads():1188] run started: s9hpsjf9 with start time 1733884838.452824 +2024-12-10 21:40:38,737 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:40:38,737 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:40:38,787 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:40:38,792 DEBUG HandlerThread:608881 [system_info.py:__init__():26] System info init +2024-12-10 21:40:38,792 DEBUG HandlerThread:608881 [system_info.py:__init__():41] System info init done +2024-12-10 21:40:38,792 INFO HandlerThread:608881 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:40:38,792 INFO SystemMonitor:608881 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:40:38,792 INFO HandlerThread:608881 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:40:38,793 INFO SystemMonitor:608881 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:40:38,793 INFO SystemMonitor:608881 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:40:38,794 INFO SystemMonitor:608881 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:40:38,794 INFO SystemMonitor:608881 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:40:38,795 INFO SystemMonitor:608881 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:40:38,805 DEBUG HandlerThread:608881 [system_info.py:probe():152] Probing system +2024-12-10 21:40:38,807 DEBUG HandlerThread:608881 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_info.py:probe():200] Probing system done +2024-12-10 21:40:38,811 DEBUG HandlerThread:608881 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:40:38.805785', 'startedAt': '2024-12-11T02:40:38.443244', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3754.252, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4503.722, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 402.6240234375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:40:38,812 INFO HandlerThread:608881 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:40:38,812 INFO HandlerThread:608881 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:40:38,812 DEBUG HandlerThread:608881 [system_info.py:_save_code():44] Saving code +2024-12-10 21:40:38,813 DEBUG HandlerThread:608881 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:40:38,813 DEBUG HandlerThread:608881 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:40:38,875 DEBUG HandlerThread:608881 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:40:38,875 INFO HandlerThread:608881 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:40:38,878 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:40:38,878 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:40:38,878 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:40:38,879 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:40:39,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:40:39,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:40:39,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:40:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:40:39,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:39,154 INFO HandlerThread:608881 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/tb" + save: true +} +control { + relay_id: "59e8947623744db792dc242bf1b20464" +} +_info { + stream_id: "s9hpsjf9" +} + +2024-12-10 21:40:39,154 DEBUG HandlerThread:608881 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:40:39,157 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/r8eyuers-code/FISH/eval_robot.py +2024-12-10 21:40:39,170 INFO wandb-upload_0:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/aqm9dddc-wandb-metadata.json +2024-12-10 21:40:39,186 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/h7dhr8h6-diff.patch +2024-12-10 21:40:39,187 DEBUG SenderThread:608881 [sender.py:send():379] send: telemetry +2024-12-10 21:40:39,204 DEBUG SenderThread:608881 [sender.py:send():379] send: tbrecord +2024-12-10 21:40:39,205 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:40:39,206 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 with policy live +2024-12-10 21:40:39,437 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/0eq0y07d-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt +2024-12-10 21:40:39,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:40:39,725 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH +2024-12-10 21:40:39,725 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code +2024-12-10 21:40:40,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:41,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:41,723 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:42,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:43,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:43,447 DEBUG SenderThread:608881 [sender.py:send():379] send: config +2024-12-10 21:40:43,490 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:44,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:45,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:45,724 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:46,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:47,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:48,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:49,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:49,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:49,726 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:50,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:51,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:52,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:53,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:53,727 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:40:54,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:40:54,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:40:54,149 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:54,189 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:40:55,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:56,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:57,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:58,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:59,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:40:59,728 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:00,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:00,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:01,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:02,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:03,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:03,729 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:04,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:05,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:05,188 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:05,730 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:06,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:07,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:07,731 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:08,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:09,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:09,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:09,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:09,731 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:10,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:10,298 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:10,732 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml +2024-12-10 21:41:11,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:11,732 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:12,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:13,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:13,733 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:14,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:15,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:15,494 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:15,733 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:16,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:17,113 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:17,734 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:18,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:19,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:19,734 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:20,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:20,513 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:21,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:21,735 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:22,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:23,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:23,735 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:24,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:24,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:24,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:25,114 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:25,707 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:25,736 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:26,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:27,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:27,737 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:28,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:29,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:29,561 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:41:29,561 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 with policy now +2024-12-10 21:41:29,561 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:41:29,562 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:41:29,562 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:41:29,563 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos +2024-12-10 21:41:29,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media +2024-12-10 21:41:29,827 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/35bhfxg3-media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:30,108 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/oa60o3l4-media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:41:30,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:31,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:31,567 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:31,738 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:32,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:33,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:33,739 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:34,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:35,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:36,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:37,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:37,122 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:37,740 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:38,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:38,795 DEBUG SystemMonitor:608881 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:41:38,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:41:39,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:39,107 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:39,149 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:40,116 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:41,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:41,742 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:41:42,003 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/mz351q7u-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:41:42,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:42,229 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:43,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:43,743 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:44,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:45,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:46,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:47,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:47,234 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:47,744 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:48,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:49,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:49,745 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:50,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:51,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:51,745 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:52,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:52,431 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:53,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:53,746 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:54,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:41:54,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:41:54,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:55,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:55,747 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:56,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:57,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:57,451 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:41:57,748 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:41:58,119 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:59,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:41:59,748 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:00,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:01,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:01,749 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:02,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:02,646 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:03,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:03,750 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:04,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:05,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:05,750 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:06,120 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:07,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:07,751 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:07,837 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:08,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:08,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:42:09,107 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:09,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:09,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:09,752 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:10,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:11,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:11,752 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:12,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:13,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:13,257 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:13,753 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:13,856 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:42:13,856 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 with policy now +2024-12-10 21:42:13,859 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:42:13,859 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:42:13,859 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:42:13,860 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:42:14,124 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:14,131 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/rsee72h5-media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:14,753 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:42:14,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:14,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:42:15,088 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/4wjshnhu-media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:42:15,124 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:15,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:16,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:16,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:17,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:17,754 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:18,125 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:18,926 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:19,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:20,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:21,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:21,756 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:22,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:23,127 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:24,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:24,108 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:24,150 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:24,263 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:25,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:26,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:27,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:28,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:29,128 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:30,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:30,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:31,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:32,129 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:33,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:34,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:35,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:35,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:36,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:37,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:38,132 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:38,796 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:42:39,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:39,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:40,131 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:40,246 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:41,130 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:42,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:43,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:44,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:45,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:46,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:46,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:47,134 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:48,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:48,612 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:42:48,612 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:42:48,614 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:42:48,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:42:48,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:42:49,018 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/dfwzg48s-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:42:49,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:49,764 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:50,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:51,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:51,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:52,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:53,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:53,766 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:54,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:42:54,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:42:54,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:55,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:55,767 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:56,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:56,203 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:42:57,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:57,767 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:42:58,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:59,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:42:59,768 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:00,136 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:01,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:01,399 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:01,769 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:02,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:03,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:03,770 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:04,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:05,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:05,770 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:06,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:06,439 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:07,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:07,771 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:08,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:08,797 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:43:09,108 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:09,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:09,137 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:09,772 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:10,138 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:11,138 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:11,596 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:11,772 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:12,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:13,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:13,773 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:14,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:15,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:15,774 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:16,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:16,612 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:17,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:17,775 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:18,139 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:19,140 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:19,775 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:20,142 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:20,417 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:43:20,418 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:43:20,418 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_aac838f24394620216df.mp4 with policy now +2024-12-10 21:43:20,419 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:43:20,419 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:43:20,419 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:43:20,700 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/srqd6xbw-media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:20,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:43:21,087 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/739rtuu0-media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:43:21,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:21,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:22,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:22,435 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:22,776 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:23,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:24,117 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:24,117 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:24,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:25,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:25,777 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:26,143 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:27,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:27,778 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:28,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:28,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:29,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:29,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:43:30,144 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:30,444 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:43:30,444 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:43:30,445 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:43:30,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:43:31,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:31,779 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:32,153 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:33,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:33,402 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:34,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:35,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:35,781 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:36,145 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:37,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:37,781 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:38,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:38,516 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:38,797 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:43:39,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:39,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:39,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:39,782 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:40,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:41,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:41,783 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:42,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:43,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:43,723 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:43,784 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:44,181 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:45,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:45,785 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:46,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:47,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:47,785 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:48,146 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:48,919 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:49,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:49,786 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:50,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:51,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:51,787 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:52,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:53,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:53,787 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:53,921 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:54,109 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:43:54,109 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:43:54,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:55,147 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:55,788 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:56,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:57,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:57,789 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:43:58,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:59,115 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:43:59,157 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:43:59,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:00,148 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:01,151 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:01,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:02,150 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:02,301 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:44:02,302 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:44:02,302 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 with policy now +2024-12-10 21:44:02,302 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:02,303 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:02,303 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:02,586 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/by1wmb7v-media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:02,790 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:02,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:02,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:44:03,110 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/nincuu1n-media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:03,155 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:03,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:04,118 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:04,154 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:04,791 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:05,181 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:05,792 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:06,160 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:07,170 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:08,158 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:08,798 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:09,111 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:09,111 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:09,161 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:09,264 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:09,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:10,162 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:10,489 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:10,489 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:10,490 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:10,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:10,793 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:11,073 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/mo50gl5v-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:11,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:11,794 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:12,174 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:13,165 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:13,794 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:14,185 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:14,343 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:15,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:15,795 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:16,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:17,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:17,796 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:18,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:19,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:19,437 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:19,797 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:20,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:21,166 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:21,797 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:22,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:23,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:23,798 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:24,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:24,110 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:24,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:24,627 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:25,167 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:25,799 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:26,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:27,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:27,800 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:28,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:29,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:29,647 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:29,800 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:30,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:31,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:31,801 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:32,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:33,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:33,802 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:34,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:34,663 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:35,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:35,803 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:36,168 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:37,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:37,803 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:38,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:38,798 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:39,110 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:44:39,113 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:44:39,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:39,669 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:39,804 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:40,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:41,169 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:41,805 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:42,172 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:43,177 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:43,249 DEBUG SenderThread:608881 [sender.py:send():379] send: files +2024-12-10 21:44:43,249 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 with policy now +2024-12-10 21:44:43,251 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:44:43,252 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:43,252 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:43,252 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:43,556 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/09qsiei9-media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:43,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval +2024-12-10 21:44:44,132 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/e8z9uoxa-media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:44,176 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:44,806 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:45,121 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:45,179 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:46,185 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:47,191 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:47,807 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:48,183 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:49,187 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:49,808 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:50,126 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:50,188 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:44:50,962 DEBUG SenderThread:608881 [sender.py:send():379] send: exit +2024-12-10 21:44:50,962 INFO SenderThread:608881 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:44:50,962 INFO SenderThread:608881 [sender.py:send_exit():588] handling runtime: 252 +2024-12-10 21:44:50,965 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:50,965 INFO SenderThread:608881 [sender.py:send_exit():594] send defer +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:44:50,966 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:44:50,966 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:44:50,966 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:44:50,966 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:44:50,966 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:44:50,967 DEBUG SystemMonitor:608881 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:44:50,967 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:44:50,967 DEBUG SystemMonitor:608881 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:44:51,015 INFO HandlerThread:608881 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send():379] send: stats +2024-12-10 21:44:51,016 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:51,016 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:44:51,016 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:44:51,016 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:44:51,016 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:51,016 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:44:51,808 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:51,809 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:51,809 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:57,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:57,018 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,019 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:44:57,019 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:44:57,019 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:57,019 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,019 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:57,019 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:44:57,020 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send():379] send: history +2024-12-10 21:44:57,020 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,024 DEBUG SenderThread:608881 [sender.py:send():379] send: summary +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:44:57,024 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:44:57,024 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:44:57,024 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,024 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:44:57,025 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:44:57,025 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:44:57,025 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,025 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:44:57,025 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,025 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:44:57,810 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:44:57,963 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:57,990 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:44:57,990 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:57,990 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:57,991 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:44:57,991 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:57,991 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:44:57,991 INFO SenderThread:608881 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:44:57,993 INFO SenderThread:608881 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:44:58,007 INFO SenderThread:608881 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:44:58,022 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:44:58,023 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:58,023 DEBUG SenderThread:608881 [sender.py:send():379] send: artifact +2024-12-10 21:44:58,023 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:44:58,597 INFO wandb-upload_0:608881 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5dl4h77g +2024-12-10 21:44:58,597 INFO wandb-upload_2:608881 [upload_job.py:push():85] Skipped uploading /tmp/tmpob1p2z0k/wandb-job.json +2024-12-10 21:44:58,811 INFO Thread-12 :608881 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:44:58,861 INFO wandb-upload_1:608881 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpjyl59zur +2024-12-10 21:44:58,963 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:59,526 INFO SenderThread:608881 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NjgxMjE3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NjgwNjQxOA==', 'versionIndex': 343}}} +2024-12-10 21:44:59,526 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:59,526 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:44:59,526 INFO SenderThread:608881 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:44:59,811 INFO SenderThread:608881 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml config.yaml +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json wandb-summary.json +2024-12-10 21:44:59,812 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt requirements.txt +2024-12-10 21:44:59,813 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:44:59,815 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:44:59,816 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log output.log +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/diff.patch diff.patch +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 +2024-12-10 21:44:59,817 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 +2024-12-10 21:44:59,818 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 media/videos/eval/2_eval_2_aac838f24394620216df.mp4 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:44:59,820 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:44:59,820 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:44:59,820 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:44:59,820 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:44:59,820 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:44:59,820 INFO SenderThread:608881 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:44:59,964 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:44:59,964 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,000 INFO wandb-upload_2:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/requirements.txt +2024-12-10 21:45:00,030 INFO wandb-upload_1:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/wandb-summary.json +2024-12-10 21:45:00,058 INFO wandb-upload_0:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/config.yaml +2024-12-10 21:45:00,095 INFO wandb-upload_4:608881 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/files/output.log +2024-12-10 21:45:00,103 INFO wandb-upload_3:608881 [upload_job.py:push():130] Uploaded file /tmp/tmp0jm5xe66wandb/j6opz80u-events.out.tfevents.1733884839.leonmkim-ROG-Strix-G15CS-G15CS.608694.0 +2024-12-10 21:45:00,303 INFO Thread-11 (_thread_body):608881 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:45:00,303 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,303 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:45:00,304 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,304 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:45:00,304 INFO SenderThread:608881 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:45:00,305 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:45:00,305 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,305 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:45:00,305 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,305 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:45:00,305 INFO SenderThread:608881 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:45:00,501 INFO SenderThread:608881 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:45:00,502 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,502 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:45:00,502 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:45:00,502 INFO SenderThread:608881 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:45:00,502 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:45:00,502 INFO HandlerThread:608881 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:45:00,502 DEBUG SenderThread:608881 [sender.py:send():379] send: final +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send():379] send: footer +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: defer +2024-12-10 21:45:00,503 INFO SenderThread:608881 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:45:00,503 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,503 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:45:00,503 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:45:00,505 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:45:00,506 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:45:00,560 DEBUG SenderThread:608881 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:45:00,560 INFO MainThread:608881 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:45:00,560 DEBUG HandlerThread:608881 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:45:00,560 INFO HandlerThread:608881 [handler.py:finish():882] shutting down handler +2024-12-10 21:45:01,505 INFO WriterThread:608881 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb +2024-12-10 21:45:01,560 INFO SenderThread:608881 [sender.py:finish():1615] shutting down sender +2024-12-10 21:45:01,560 INFO SenderThread:608881 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:45:01,560 INFO SenderThread:608881 [file_pusher.py:join():175] waiting for file pusher diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug.log b/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c03fbf1338eb8bfef5e144c1346b71a6b1d55224 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Configure stats pid to 608694 +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:40:38,444 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/settings +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug.log +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/wandb/run-20241210_214038-s9hpsjf9/logs/debug-internal.log +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:40:38,445 INFO MainThread:608694 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:40:38,446 INFO MainThread:608694 [wandb_init.py:init():619] starting backend +2024-12-10 21:40:38,446 INFO MainThread:608694 [wandb_init.py:init():623] setting up manager +2024-12-10 21:40:38,451 INFO MainThread:608694 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:40:38,452 INFO MainThread:608694 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:40:38,464 INFO MainThread:608694 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:40:38,471 INFO MainThread:608694 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:40:38,736 INFO MainThread:608694 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:40:38,778 INFO MainThread:608694 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:40:38,780 INFO MainThread:608694 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:40:39,107 INFO MainThread:608694 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:40:39,108 INFO MainThread:608694 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:40:39,109 INFO MainThread:608694 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:40:39,110 INFO MainThread:608694 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/214032/tb, True +2024-12-10 21:40:43,446 INFO MainThread:608694 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:45:01,564 WARNING MsgRouterThr:608694 [router.py:message_loop():77] message_loop has been closed diff --git a/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb b/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..eb52b04489858c81c066a523ff9becc34ab843f5 --- /dev/null +++ b/214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9065372c87211d1d86104cd48cb74c0d14f6ce2030f56f4b970f2d078da567c5 +size 290992 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e579187c65ce14e2833b02e62b4984940205a0eb --- /dev/null +++ b/config.yaml @@ -0,0 +1,453 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '265626' +wandb_notes: 265626_req_265827 +eval: false +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- hbm_twodim_left_to_right +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 0 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c771f63b84b76a8403ed7cfdf10b8d6df0ace33 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8bf36ece557bb4c8411abf261b04c1027f14173c566c0c85a8f34b3813d8349 +size 910102870 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..b29afc01bb618d783969c73a45663466ab81e3dc --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178a65e76f73350fc7fff2fa02d2fbc3c886667cf963725b26c97a4573947de7 +size 910102870 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..41281da7e06da538d81fb6122078e8f94cc9105a --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ea4da60ac4a9ad63343f5f777c4a77a92cd0710b4ee2023f3d8c954ab72776 +size 910102360 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..382940a26bea434caacb69ab2ee67c8e7de251b5 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fffaf78b53eac9b6329a18a4e90edfd7797613d901b2aed4c942706d7af7b70 +size 910102870 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..adf9f6cd0206105595df68bd8db245f777987c4a --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba5e8a28fe32378b1da70d0327418ae71178eb639a6c5d5fde66ab2e4dee49c +size 910102360 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..382940a26bea434caacb69ab2ee67c8e7de251b5 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fffaf78b53eac9b6329a18a4e90edfd7797613d901b2aed4c942706d7af7b70 +size 910102870