diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..1abe9e46b3240ed3dbc590a30310846fdc8c1a51 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 filter=lfs diff=lfs merge=lfs -text +210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 filter=lfs diff=lfs merge=lfs -text +210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 filter=lfs diff=lfs merge=lfs -text +210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 filter=lfs diff=lfs merge=lfs -text +210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 filter=lfs diff=lfs merge=lfs -text +210705/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210705/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210705/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210705/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210705/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210705/episode_rosbags/episode_3_2024-12-29-21-10-01.bag filter=lfs diff=lfs merge=lfs -text +210705/episode_rosbags/episode_4_2024-12-29-21-10-51.bag filter=lfs diff=lfs merge=lfs -text +210705/episode_rosbags/episode_2_2024-12-29-21-09-13.bag filter=lfs diff=lfs merge=lfs -text +210705/episode_rosbags/episode_0_2024-12-29-21-07-33.bag filter=lfs diff=lfs merge=lfs -text +210705/episode_rosbags/episode_1_2024-12-29-21-08-12.bag filter=lfs diff=lfs merge=lfs -text diff --git a/210700/.hydra/config.yaml b/210700/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b0dbb647ec55f19e7dcfda5b19262d0a6e5a197 --- /dev/null +++ b/210700/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1051_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/210700/.hydra/hydra.yaml b/210700/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56059eb5852813d47e7120ecd7214de96757fc3b --- /dev/null +++ b/210700/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210700 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/210700/.hydra/overrides.yaml b/210700/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/210700/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/210700/eval_policy.log b/210700/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..4e4ca54efb9c940a09bb35d405c295bbb26f6e80 --- /dev/null +++ b/210700/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-29 21:07:00,892][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 21:07:00,896][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-29 21:07:04,575][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/210705/.hydra/config.yaml b/210705/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b0dbb647ec55f19e7dcfda5b19262d0a6e5a197 --- /dev/null +++ b/210705/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1051_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/210705/.hydra/hydra.yaml b/210705/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5aa4a4a79e80ec8fb75065880a18cf4eec959ab5 --- /dev/null +++ b/210705/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/210705/.hydra/overrides.yaml b/210705/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/210705/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/210705/episode_rosbags/aligned_depth_to_color_K.npy b/210705/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210705/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210705/episode_rosbags/cam_tf_world.npy b/210705/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/210705/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/210705/episode_rosbags/color_K.npy b/210705/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210705/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210705/episode_rosbags/depth_K.npy b/210705/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/210705/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/210705/episode_rosbags/episode_0_2024-12-29-21-07-33.bag b/210705/episode_rosbags/episode_0_2024-12-29-21-07-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..1b4ca1fb139b66b27aa33963c66845077e6681c4 --- /dev/null +++ b/210705/episode_rosbags/episode_0_2024-12-29-21-07-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285a89f6e6345cd674827691e77aad1df2403b9979ece5180e6fe1e2a56579e0 +size 1326324843 diff --git a/210705/episode_rosbags/episode_1_2024-12-29-21-08-12.bag b/210705/episode_rosbags/episode_1_2024-12-29-21-08-12.bag new file mode 100644 index 0000000000000000000000000000000000000000..b457fd6c47b6d45c8f7555208422ba44911c2337 --- /dev/null +++ b/210705/episode_rosbags/episode_1_2024-12-29-21-08-12.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75b7f0b8abf7523eb1cd9d8ca456a44478dbb74525e457ed61ec627a321bb794 +size 1328139573 diff --git a/210705/episode_rosbags/episode_2_2024-12-29-21-09-13.bag b/210705/episode_rosbags/episode_2_2024-12-29-21-09-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..c55efac45a28fb8b29edcfc47272ef4661397ed6 --- /dev/null +++ b/210705/episode_rosbags/episode_2_2024-12-29-21-09-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:830aef74ef4b8550ea3c2b6d1ea05992ba47c5527a20e1a1643861da87c97344 +size 1330391729 diff --git a/210705/episode_rosbags/episode_3_2024-12-29-21-10-01.bag b/210705/episode_rosbags/episode_3_2024-12-29-21-10-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..545e48c2f8e7e2028e3a2f17cf88f95d33ac70c3 --- /dev/null +++ b/210705/episode_rosbags/episode_3_2024-12-29-21-10-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d2cdd951418865af2c6688a2bd25af46a2b90236d38775b8aef328bb6e1f49 +size 1329523148 diff --git a/210705/episode_rosbags/episode_4_2024-12-29-21-10-51.bag b/210705/episode_rosbags/episode_4_2024-12-29-21-10-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..6c80f62af850936484f9291614ec2763fb41220a --- /dev/null +++ b/210705/episode_rosbags/episode_4_2024-12-29-21-10-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3269b873c5f8f2a888636942683099d50708b4bff863d97bbb5d456dc0a3482c +size 1329613891 diff --git a/210705/eval_robot.log b/210705/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c91d46544fdcc4f93c9b6e557dc19192aaaa1b45 --- /dev/null +++ b/210705/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-29 21:07:05,513][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 21:07:05,516][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/210705/eval_video/0_eval.mp4 b/210705/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3cb93fb4e80237ee6870e5b722493b24280dbe35 --- /dev/null +++ b/210705/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4be4b57f794bee20f5564992ee67b022ba0fa05153f2acfd6247f5c01fe68a2 +size 1151332 diff --git a/210705/eval_video/1_eval.mp4 b/210705/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa950298dde07b471dff9675c3df2b38130f147f --- /dev/null +++ b/210705/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a107c4c62341f47056320339f391b4ba52de893df4146a8b852cfd2bdd2a38b6 +size 1154425 diff --git a/210705/eval_video/2_eval.mp4 b/210705/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018f1a9a3f0c80287ae1a2d4a65dc1d8ff34a11d --- /dev/null +++ b/210705/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:639dd4a333330f56b6d93beff55c8df46b775713592e90759c34d40aebabf5c0 +size 1146794 diff --git a/210705/eval_video/3_eval.mp4 b/210705/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14138cf58264ccf109952830d27896e165190257 --- /dev/null +++ b/210705/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be2fc636735f4996c9dcfc0866e7a227637cc528400f898e5df37430c69321b +size 1159865 diff --git a/210705/eval_video/4_eval.mp4 b/210705/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..579b0c81f50be220cede74c261597e8e43630501 --- /dev/null +++ b/210705/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6559ba197b6fb10731c066ed282e33da3f63cd3ff933d4f145c53d0647693886 +size 1164276 diff --git a/210705/tb/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 b/210705/tb/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 new file mode 100644 index 0000000000000000000000000000000000000000..c7da526ad72e68b136cd85e511706e0b859313ed --- /dev/null +++ b/210705/tb/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0507d758444468ed66b691d71ab7a3bc06b63218bf66a1976dabd7a5247e074 +size 1103 diff --git a/210705/wandb/debug-internal.log b/210705/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..869c2cd552e7b6dbf21fb484f8c6839386d6a6c3 --- /dev/null +++ b/210705/wandb/debug-internal.log @@ -0,0 +1,730 @@ +2024-12-29 21:07:11,464 INFO StreamThr :1357810 [internal.py:wandb_internal():85] W&B internal server running at pid: 1357810, started at: 2024-12-29 21:07:11.463909 +2024-12-29 21:07:11,465 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status +2024-12-29 21:07:11,479 INFO WriterThread:1357810 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb +2024-12-29 21:07:11,479 DEBUG SenderThread:1357810 [sender.py:send():379] send: header +2024-12-29 21:07:11,486 DEBUG SenderThread:1357810 [sender.py:send():379] send: run +2024-12-29 21:07:11,756 INFO SenderThread:1357810 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files +2024-12-29 21:07:11,756 INFO SenderThread:1357810 [sender.py:_start_run_threads():1188] run started: amv7lra0 with start time 1735524431.464383 +2024-12-29 21:07:11,772 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 21:07:11,772 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: check_version +2024-12-29 21:07:11,851 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 21:07:11,855 DEBUG HandlerThread:1357810 [system_info.py:__init__():26] System info init +2024-12-29 21:07:11,855 DEBUG HandlerThread:1357810 [system_info.py:__init__():41] System info init done +2024-12-29 21:07:11,855 INFO HandlerThread:1357810 [system_monitor.py:start():194] Starting system monitor +2024-12-29 21:07:11,856 INFO SystemMonitor:1357810 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 21:07:11,856 INFO HandlerThread:1357810 [system_monitor.py:probe():214] Collecting system info +2024-12-29 21:07:11,856 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 21:07:11,857 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started disk monitoring +2024-12-29 21:07:11,857 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 21:07:11,858 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started memory monitoring +2024-12-29 21:07:11,859 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started network monitoring +2024-12-29 21:07:11,870 DEBUG HandlerThread:1357810 [system_info.py:probe():152] Probing system +2024-12-29 21:07:11,871 DEBUG HandlerThread:1357810 [system_info.py:_probe_git():137] Probing git +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:_probe_git():145] Probing git done +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:probe():200] Probing system done +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T02:07:11.870193', 'startedAt': '2024-12-30T02:07:11.455436', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3567.4849999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4513.316, 'min': 800.0, 'max': 4700.0}, {'current': 4513.77, 'min': 800.0, 'max': 4700.0}, {'current': 4529.436, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 470.08051681518555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 21:07:11,876 INFO HandlerThread:1357810 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 21:07:11,876 INFO HandlerThread:1357810 [system_monitor.py:probe():227] Publishing system info +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:_save_code():44] Saving code +2024-12-29 21:07:11,877 DEBUG HandlerThread:1357810 [system_info.py:_save_code():67] Saving code done +2024-12-29 21:07:11,878 DEBUG HandlerThread:1357810 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 21:07:11,940 DEBUG HandlerThread:1357810 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 21:07:11,940 INFO HandlerThread:1357810 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 21:07:11,943 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:07:11,943 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 21:07:11,944 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 21:07:11,944 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 21:07:12,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 21:07:12,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: python_packages +2024-12-29 21:07:12,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:12,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:12,175 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:12,196 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/uvy85ybf-wandb-metadata.json +2024-12-29 21:07:12,203 INFO wandb-upload_2:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/r4am14fr-diff.patch +2024-12-29 21:07:12,212 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/3ffkt5gr-code/FISH/eval_robot.py +2024-12-29 21:07:12,217 INFO HandlerThread:1357810 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/tb" + save: true +} +control { + relay_id: "976e5e9ee00945adac42fe84b7ff337e" +} +_info { + stream_id: "amv7lra0" +} + +2024-12-29 21:07:12,217 DEBUG HandlerThread:1357810 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 21:07:12,230 DEBUG SenderThread:1357810 [sender.py:send():379] send: telemetry +2024-12-29 21:07:12,266 DEBUG SenderThread:1357810 [sender.py:send():379] send: tbrecord +2024-12-29 21:07:12,267 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:07:12,267 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 with policy live +2024-12-29 21:07:12,446 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/g38txylz-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:07:12,757 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH +2024-12-29 21:07:13,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:14,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:14,756 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:15,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:16,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:17,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:17,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:17,277 DEBUG SenderThread:1357810 [sender.py:send():379] send: config +2024-12-29 21:07:18,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:18,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:19,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:20,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:20,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:21,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:22,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:22,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:22,759 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:23,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:24,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:24,760 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:25,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:26,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:26,760 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:27,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:27,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:27,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:27,329 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:28,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:29,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:30,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:30,761 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:31,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:32,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:32,762 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:33,055 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:33,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:34,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:34,763 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:35,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:36,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:37,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:38,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:38,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:39,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:40,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:40,764 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:41,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:42,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:42,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:42,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:42,765 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:43,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:43,306 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:43,765 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml +2024-12-29 21:07:44,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:45,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:46,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:46,766 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:47,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:48,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:49,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:49,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:50,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:51,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:52,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:53,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:54,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:54,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:55,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:56,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:57,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:57,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:57,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:58,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:59,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:59,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:00,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:00,807 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:08:00,807 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 with policy now +2024-12-29 21:08:00,808 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:08:00,809 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:08:00,809 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:08:00,812 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:08:01,066 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/xmtm0m8l-media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:01,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media +2024-12-29 21:08:01,771 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos +2024-12-29 21:08:01,771 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:08:02,065 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/eul3dkol-media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:02,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:02,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:03,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:04,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:04,413 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:05,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:06,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:06,772 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:07,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:08,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:08,772 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:09,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:09,773 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:08:10,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:10,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:10,773 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:11,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:11,859 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 21:08:11,859 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:08:12,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:12,174 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:12,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:12,774 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:13,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:14,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:14,775 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:15,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:16,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:16,249 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:17,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:18,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:19,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:20,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:21,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:22,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:22,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:23,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:24,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:25,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:26,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:27,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:27,174 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:27,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:27,256 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:28,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:29,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:30,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:31,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:32,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:33,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:33,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:34,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:35,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:36,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:37,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:38,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:38,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:39,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:40,189 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:40,784 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:40,858 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:08:40,858 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 with policy now +2024-12-29 21:08:40,859 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:08:40,859 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:08:40,859 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:08:40,860 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:08:41,106 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/alfog98n-media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:41,191 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:41,784 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:08:41,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:41,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:08:41,860 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:08:42,061 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/b2tkslf7-media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:42,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:42,181 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:42,225 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:42,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:43,192 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:43,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:44,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:45,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:46,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:46,786 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:47,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:48,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:48,787 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:49,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:49,305 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:50,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:51,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:52,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:53,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:54,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:55,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:55,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:56,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:57,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:57,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:57,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:58,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:59,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:00,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:00,304 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:01,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:02,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:03,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:04,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:05,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:06,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:06,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:07,200 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:08,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:09,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:10,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:10,633 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:10,633 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:10,634 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:10,794 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:10,794 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:11,032 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/4pzrw60x-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:11,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:11,860 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:09:11,861 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:12,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:12,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:12,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:12,795 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:13,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:14,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:14,795 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:15,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:16,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:16,796 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:17,014 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:17,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:18,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:19,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:20,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:21,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:22,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:22,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:23,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:24,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:25,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:26,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:27,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:27,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:27,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:27,296 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:28,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:29,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:30,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:31,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:32,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:33,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:33,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:34,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:35,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:36,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:37,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:38,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:39,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:39,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:40,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:40,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:41,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:41,555 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:09:41,555 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 with policy now +2024-12-29 21:09:41,558 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:09:41,558 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:41,558 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:41,569 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:41,796 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/6mdf4ynb-media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:41,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:41,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:41,805 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:09:41,861 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:09:42,056 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/2h3qfory-media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:42,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:42,182 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:42,229 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:42,805 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:43,210 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:44,211 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:44,259 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:45,229 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:46,213 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:46,806 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:47,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:48,212 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:49,220 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:50,005 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:50,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:50,807 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:51,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:52,213 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:53,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:54,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:55,191 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:55,223 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:56,219 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:57,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:57,186 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:57,233 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:58,225 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:58,470 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:58,471 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:58,471 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:58,810 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:58,810 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:59,231 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:00,224 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:00,811 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:01,121 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:01,226 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:02,240 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:02,812 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:03,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:04,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:04,813 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:05,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:06,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:06,294 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:07,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:08,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:09,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:10,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:11,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:11,861 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:10:11,862 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:12,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:12,183 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:12,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:13,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:14,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:15,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:16,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:17,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:17,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:18,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:19,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:20,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:21,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:22,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:23,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:23,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:24,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:25,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:26,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:27,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:27,184 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:27,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:28,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:28,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:29,245 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:30,248 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:30,264 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:10:30,264 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 with policy now +2024-12-29 21:10:30,305 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:10:30,306 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:10:30,306 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:10:30,307 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:10:30,518 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/dcpthqn9-media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:30,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:10:31,087 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/ig4s70k7-media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:31,246 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:32,251 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:32,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:33,250 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:33,911 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:34,262 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:34,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:35,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:36,255 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:37,251 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:38,254 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:38,824 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:39,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:39,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:40,255 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:41,259 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:41,862 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:10:42,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:42,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:42,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:43,260 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:44,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:44,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:45,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:46,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:47,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:47,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:10:48,072 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/83t2lcan-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:10:48,262 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:48,526 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:10:48,526 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:10:48,526 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:10:48,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:10:48,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:49,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:49,645 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:50,273 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:50,828 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:51,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:52,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:52,828 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:53,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:54,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:55,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:55,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:56,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:57,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:57,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:57,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:58,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:59,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:00,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:00,715 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:00,831 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:01,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:02,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:02,832 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:03,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:04,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:04,833 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:05,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:05,716 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:06,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:07,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:08,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:08,834 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:09,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:10,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:10,916 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:11,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:11,862 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:12,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:12,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:12,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:13,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:14,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:15,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:16,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:16,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:17,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:18,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:19,283 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:19,553 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:11:19,553 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 with policy now +2024-12-29 21:11:19,556 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:11:19,556 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:19,556 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:19,557 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:19,803 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/m56r65qv-media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:11:20,087 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/si2ao02c-media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:20,298 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:20,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:21,288 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:21,568 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:22,300 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:23,303 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:24,300 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:24,840 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:25,306 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:26,303 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:27,137 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:27,192 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:27,193 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:27,302 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:28,327 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:28,841 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:29,308 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:30,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:31,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:32,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:32,307 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:33,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:34,324 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:35,313 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:36,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:37,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:37,325 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:38,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:38,585 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:38,585 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:38,585 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:38,844 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:38,844 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:11:38,845 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:39,317 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:40,320 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:41,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:41,863 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:42,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:42,190 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:42,304 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:42,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:43,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:44,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:45,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:46,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:47,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:48,219 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:48,324 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:48,901 DEBUG SenderThread:1357810 [sender.py:send():379] send: exit +2024-12-29 21:11:48,901 INFO SenderThread:1357810 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 21:11:48,901 INFO SenderThread:1357810 [sender.py:send_exit():588] handling runtime: 277 +2024-12-29 21:11:48,902 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:48,902 INFO SenderThread:1357810 [sender.py:send_exit():594] send defer +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 21:11:48,903 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 1 +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 21:11:48,903 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 2 +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 21:11:48,903 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 21:11:48,904 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 21:11:48,904 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 21:11:48,904 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 21:11:48,953 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined network monitor +2024-12-29 21:11:48,954 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,954 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 21:11:48,954 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 3 +2024-12-29 21:11:48,954 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:48,954 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 21:11:48,955 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,955 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 21:11:48,955 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 4 +2024-12-29 21:11:48,955 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,955 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 21:11:49,848 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:50,848 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:55,586 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 21:11:55,586 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 21:11:55,587 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 5 +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:55,587 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:55,587 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:55,588 DEBUG SenderThread:1357810 [sender.py:send():379] send: summary +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:55,588 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 6 +2024-12-29 21:11:55,588 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,589 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 21:11:55,589 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 7 +2024-12-29 21:11:55,589 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:55,589 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,589 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 21:11:55,589 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 21:11:55,850 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:55,902 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:58,205 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 8 +2024-12-29 21:11:58,205 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:58,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:58,205 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 21:11:58,206 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:58,206 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 21:11:58,206 INFO SenderThread:1357810 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 21:11:58,208 INFO SenderThread:1357810 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 21:11:58,222 INFO SenderThread:1357810 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 21:11:58,230 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 9 +2024-12-29 21:11:58,231 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:58,231 DEBUG SenderThread:1357810 [sender.py:send():379] send: artifact +2024-12-29 21:11:58,231 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 21:11:58,685 INFO wandb-upload_2:1357810 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8w1_xu6h +2024-12-29 21:11:58,851 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:58,865 INFO wandb-upload_0:1357810 [upload_job.py:push():88] Uploaded file /tmp/tmpg1hqer99/wandb-job.json +2024-12-29 21:11:58,867 INFO wandb-upload_1:1357810 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4rwvgu1e +2024-12-29 21:11:58,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA2MzIwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA1NjUzOQ==', 'versionIndex': 386}}} +2024-12-29 21:11:59,443 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 21:11:59,851 INFO SenderThread:1357810 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml config.yaml +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json wandb-summary.json +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt requirements.txt +2024-12-29 21:11:59,855 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json wandb-metadata.json +2024-12-29 21:11:59,855 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:11:59,856 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log output.log +2024-12-29 21:11:59,856 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch diff.patch +2024-12-29 21:11:59,857 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 21:11:59,857 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:11:59,859 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 10 +2024-12-29 21:11:59,860 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:59,861 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:59,861 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 21:11:59,861 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:59,861 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 21:11:59,861 INFO SenderThread:1357810 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:11:59,904 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:59,905 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,027 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:12:00,027 INFO wandb-upload_2:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml +2024-12-29 21:12:00,061 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt +2024-12-29 21:12:00,133 INFO wandb-upload_4:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:12:00,164 INFO wandb-upload_3:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/2wpcfjl2-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:12:00,364 INFO Thread-11 (_thread_body):1357810 [sender.py:transition_state():614] send defer: 11 +2024-12-29 21:12:00,364 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,364 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 21:12:00,365 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,365 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 21:12:00,365 INFO SenderThread:1357810 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 12 +2024-12-29 21:12:00,366 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,366 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 21:12:00,366 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [file_stream.py:finish():601] file stream finish called +2024-12-29 21:12:00,616 INFO SenderThread:1357810 [file_stream.py:finish():605] file stream finish is done +2024-12-29 21:12:00,616 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 13 +2024-12-29 21:12:00,616 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,616 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 21:12:00,616 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 14 +2024-12-29 21:12:00,617 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,617 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send():379] send: final +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send():379] send: footer +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 21:12:00,617 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:12:00,618 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,618 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 21:12:00,618 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: server_info +2024-12-29 21:12:00,619 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:12:00,716 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 21:12:00,717 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 21:12:00,717 INFO HandlerThread:1357810 [handler.py:finish():882] shutting down handler +2024-12-29 21:12:01,620 INFO WriterThread:1357810 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [sender.py:finish():1615] shutting down sender +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [file_pusher.py:join():175] waiting for file pusher diff --git a/210705/wandb/debug.log b/210705/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6e4c547498baf6eff48c34c93d3c4890e4d6d64f --- /dev/null +++ b/210705/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-29 21:07:11,457 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Configure stats pid to 1357631 +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/settings +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/logs/debug.log +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/logs/debug-internal.log +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:init():569] calling init triggers +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 5, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1051_0', 'wandb_notes': '1051_0_req_1368_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705'} +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():619] starting backend +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():623] setting up manager +2024-12-29 21:07:11,463 INFO MainThread:1357631 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 21:07:11,464 INFO MainThread:1357631 [wandb_init.py:init():631] backend started and connected +2024-12-29 21:07:11,478 INFO MainThread:1357631 [wandb_init.py:init():720] updated telemetry +2024-12-29 21:07:11,484 INFO MainThread:1357631 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 21:07:11,772 INFO MainThread:1357631 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 21:07:11,846 INFO MainThread:1357631 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 21:07:11,846 INFO MainThread:1357631 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 21:07:12,173 INFO MainThread:1357631 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 21:07:12,176 INFO MainThread:1357631 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 21:07:12,177 INFO MainThread:1357631 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/tb, True +2024-12-29 21:07:17,276 INFO MainThread:1357631 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 21:12:01,717 WARNING MsgRouterThr:1357631 [router.py:message_loop():77] message_loop has been closed diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py b/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml b/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc4d5445795801efd6cf1662c0b85810f7e0541e --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml @@ -0,0 +1,893 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 5 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1051_0' +wandb_notes: + desc: null + value: 1051_0_req_1368_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: false +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735524431 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch b/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..6091680f6755fb87c6aec59d3d880084437204f7 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch @@ -0,0 +1,192 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 2303343..28bd7fd 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -71,7 +71,7 @@ dtc_adaptive_normalization: false + mask_normals_within_sdf: true + adaptive_normals_mask: true + learnable_contact_preprocess_params: false +-contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' ++contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' + # contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9' + contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt' # no mask no flow, just context + # contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/197406_2/checkpoints/epoch=08-val_loss=0.00.ckpt' # w/ mask no flow, just context +@@ -123,6 +123,10 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 120 demos ++# wandb_run_id: '1045_0' # greece ++wandb_run_id: '1051_0' # all ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +@@ -144,7 +148,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '523_0' # fowlers + # wandb_run_id: '524_0' # lib + # wandb_run_id: '525_0' # modelsys +-wandb_run_id: '526_0' # electrodyn ++# wandb_run_id: '526_0' # electrodyn + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 32 held out demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 0f225ec..89242d9 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -15,8 +15,8 @@ if not local_checkpoint_root_dir.exists(): + local_checkpoint_root_dir.mkdir(parents=True) + + # Download the model checkpoints +-# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') +-remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++# remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') + remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser() + remote_username = 'leonmkim' + remote_host = 'grasp-login1' +@@ -146,6 +146,11 @@ remote_host = 'grasp-login1' + # '265627', + # '265620', + # ] ++ ++run_id_list = [ ++ '1051_0', ++] ++ + checkpoint_epoch = 12000 + + # use subprocess to download the model checkpoints in parallel +Submodule ros_ws/src/contact_estimation_ros contains modified content +diff --git a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +index dcbcfa4..35dd7e8 100755 +--- a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py ++++ b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +@@ -15,6 +15,8 @@ import time + import datetime + import shutil + ++from scipy.spatial.transform import Rotation as R ++ + # solution to launching from main thread from: https://answers.ros.org/question/260212/start-launchfile-from-service/ + # from Queue import Queue + +@@ -23,7 +25,7 @@ from fish_data_collection.srv import StartBagging, StartBaggingRequest, StartBag + from sensor_msgs.msg import CameraInfo + + class RosbaggingHandler(): +- def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, camera_calibration_dir, ++ def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, + multicam=False, align_depth_to_color=False): + ## bag files and target files should be identified by episode id + self.align_depth_to_color = align_depth_to_color +@@ -33,8 +35,6 @@ class RosbaggingHandler(): + self.intrinsics_saved = False + self.extrinsics_saved = False + +- self.camera_calibration_dir = camera_calibration_dir +- + self.full_bag_dir = os.path.join(self.base_dset_dir, self.experiment_dirname, self.demo_dirname) + os.makedirs(self.full_bag_dir, exist_ok=True) + +@@ -63,7 +63,7 @@ class RosbaggingHandler(): + self.abort_bagging_request = False + + self.save_camera_intrinsics() +- self.save_camera_extrinsics(self.camera_calibration_dir) ++ self.save_camera_extrinsics() + + while not rospy.is_shutdown(): + if self.start_bagging_request: +@@ -83,23 +83,56 @@ class RosbaggingHandler(): + self.start_bagging_request = True + return response + +- def save_camera_extrinsics(self, path_to_calibration_dir): ++ def save_camera_extrinsics(self): + if not self.extrinsics_saved: +- # copy over the contents of the calibration directory, but not the directory itself, into the bag directory +- # list the contents of the calibration directory that end in .npy +- calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) +- # calibration_dir_contents = os.listdir(path_to_calibration_dir) +- # copy the contents of the calibration directory into the bag directory +- for item in calibration_dir_contents: +- # make sure the item is a file +- if os.path.isfile(item): +- # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) +- shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) +- # also save the path to the calibration directory as a txt file +- with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: +- f.write(path_to_calibration_dir) ++ # get cam extrinsic ++ # use ros tf2 to get transform from color_optical_frame to panda_link0 ++ import tf2_ros ++ tfBuffer = tf2_ros.Buffer() ++ listener = tf2_ros.TransformListener(tfBuffer) ++ ++ # get transform from color_optical_frame to panda_link0 ++ # target, source where target is the parent frame and source is the child frame ++ color_tf_world_ros = tfBuffer.lookup_transform('camera_color_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.color_tf_world = np.eye(4) ++ self.color_tf_world[0, 3] =color_tf_world_ros.translation.x ++ self.color_tf_world[1, 3] =color_tf_world_ros.translation.y ++ self.color_tf_world[2, 3] =color_tf_world_ros.translation.z ++ self.color_tf_world[:3, :3] = R.from_quat([color_tf_world_ros.rotation.x, color_tf_world_ros.rotation.y, color_tf_world_ros.rotation.z, color_tf_world_ros.rotation.w]).as_matrix() ++ ++ # dump the info to the bag directory ++ np.save(os.path.join(self.full_bag_dir, 'color_tf_world.npy'), self.color_tf_world) ++ ++ depth_tf_world_ros = tfBuffer.lookup_transform('camera_depth_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.depth_tf_world = np.eye(4) ++ self.depth_tf_world[0, 3] = depth_tf_world_ros.translation.x ++ self.depth_tf_world[1, 3] = depth_tf_world_ros.translation.y ++ self.depth_tf_world[2, 3] = depth_tf_world_ros.translation.z ++ self.depth_tf_world[:3, :3] = \ ++ R.from_quat([ ++ depth_tf_world_ros.rotation.x, ++ depth_tf_world_ros.rotation.y, ++ depth_tf_world_ros.rotation.z, ++ depth_tf_world_ros.rotation.w, ++ ]).as_matrix() ++ ++ np.save(os.path.join(self.full_bag_dir, 'depth_tf_world.npy'), self.depth_tf_world) ++ ++ # # copy over the contents of the calibration directory, but not the directory itself, into the bag directory ++ # # list the contents of the calibration directory that end in .npy ++ # calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) ++ # # calibration_dir_contents = os.listdir(path_to_calibration_dir) ++ # # copy the contents of the calibration directory into the bag directory ++ # for item in calibration_dir_contents: ++ # # make sure the item is a file ++ # if os.path.isfile(item): ++ # # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) ++ # shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) ++ # # also save the path to the calibration directory as a txt file ++ # with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: ++ # f.write(path_to_calibration_dir) + +- self.extrinsics_saved = True ++ # self.extrinsics_saved = True + + def save_camera_intrinsics(self): + if not self.intrinsics_saved: +@@ -198,9 +231,6 @@ class RosbaggingHandler(): + if self.bag_recording: + self.current_recording_bag_path = bag_path + # response.success = True +- +- +- + # return response + + def stop_bagging_service_callback(self, req): +@@ -304,7 +334,7 @@ if __name__ == '__main__': + assert os.path.exists(rosbag_launch_path), "rosbag launch file does not exist at path: {}".format(rosbag_launch_path) + + camera_calibration_dir = '/home/leonmkim/FISH_franka_ros_ws/src/contact_estimation_ros/fish_data_collection/scripts/camera_poses_L515/20240313-153328' +- dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, camera_calibration_dir, multicam=multicam, align_depth_to_color=align_depth_to_color) ++ dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, multicam=multicam, align_depth_to_color=align_depth_to_color) + + # rospy.spin() + diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3cb93fb4e80237ee6870e5b722493b24280dbe35 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4be4b57f794bee20f5564992ee67b022ba0fa05153f2acfd6247f5c01fe68a2 +size 1151332 diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa950298dde07b471dff9675c3df2b38130f147f --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a107c4c62341f47056320339f391b4ba52de893df4146a8b852cfd2bdd2a38b6 +size 1154425 diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018f1a9a3f0c80287ae1a2d4a65dc1d8ff34a11d --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:639dd4a333330f56b6d93beff55c8df46b775713592e90759c34d40aebabf5c0 +size 1146794 diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..14138cf58264ccf109952830d27896e165190257 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be2fc636735f4996c9dcfc0866e7a227637cc528400f898e5df37430c69321b +size 1159865 diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..579b0c81f50be220cede74c261597e8e43630501 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6559ba197b6fb10731c066ed282e33da3f63cd3ff933d4f145c53d0647693886 +size 1164276 diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/output.log b/210705/wandb/run-20241229_210711-amv7lra0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ac14e2b80868f4beab23b79ec6e9d235128846c1 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/output.log @@ -0,0 +1,232 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735524437.277411]: resetting environment +[INFO] [1735524437.278617]: cleared current plan +[INFO] [1735524437.279572]: moving to home +[INFO] [1735524438.281366]: reached home +[INFO] [1735524438.282516]: reset action history +[INFO] [1735524441.040187]: environment reset +Starting episode 0 +[INFO] [1735524441.042273]: resetting environment +[INFO] [1735524441.043332]: cleared current plan +[INFO] [1735524441.044445]: moving to home +[INFO] [1735524442.046925]: reached home +[INFO] [1735524442.048198]: reset action history +[INFO] [1735524444.803746]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735524449.340801]: resetting environment +[INFO] [1735524449.341949]: cleared current plan +[INFO] [1735524449.342690]: moving to home +[INFO] [1735524450.345130]: reached home +[INFO] [1735524450.346653]: reset action history +[INFO] [1735524453.054966]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/episode_rosbags/episode_0_2024-12-29-21-07-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1357631.log +started roslaunch server http://158.130.50.37:41113/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1357930] +process[print_text-2]: started with pid [1357932] +process[rosbag_record-3]: started with pid [1357933] +started bagging! +For topic gripper_width: timestamp difference is -136483716 for nearest: 1735524458267573787 - target: 1735524458404057503 at idx 77 +For topic gripper_width: timestamp difference is -135688902 for nearest: 1735524459267566322 - target: 1735524459403255224 at idx 93 +For topic gripper_width: timestamp difference is -134891367 for nearest: 1735524460267573023 - target: 1735524460402464390 at idx 102 +For topic gripper_width: timestamp difference is -132206830 for nearest: 1735524463667568532 - target: 1735524463799775362 at idx 147 +For topic gripper_width: timestamp difference is -132050039 for nearest: 1735524463867564915 - target: 1735524463999614954 at idx 148 +For topic gripper_width: timestamp difference is -131913164 for nearest: 1735524464067541620 - target: 1735524464199454784 at idx 149 +[INFO] [1735524479.862312]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735524480.582343]: Stored episode 1. +[INFO] [1735524480.583110]: Saving video... +[INFO] [1735524480.788691]: Video saved! +Starting episode 1 +[INFO] [1735524480.809039]: resetting environment +[INFO] [1735524480.809273]: cleared current plan +[INFO] [1735524480.809620]: moving to home +[INFO] [1735524484.413071]: reached home +[INFO] [1735524484.413606]: reset action history +[INFO] [1735524487.123470]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735524489.261362]: resetting environment +[INFO] [1735524489.261593]: cleared current plan +[INFO] [1735524489.261855]: moving to home +[INFO] [1735524490.263985]: reached home +[INFO] [1735524490.264421]: reset action history +[INFO] [1735524492.970165]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/episode_rosbags/episode_1_2024-12-29-21-08-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1357631.log +started roslaunch server http://158.130.50.37:44909/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1358038] +process[print_text-5]: started with pid [1358039] +process[rosbag_record-6]: started with pid [1358040] +started bagging! +[INFO] [1735524519.719420]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735524520.481768]: Stored episode 2. +[INFO] [1735524520.482108]: Saving video... +[INFO] [1735524520.828882]: Video saved! +Starting episode 2 +[INFO] [1735524520.863370]: resetting environment +[INFO] [1735524520.871085]: cleared current plan +[INFO] [1735524520.873594]: moving to home +[INFO] [1735524524.478363]: reached home +[INFO] [1735524524.484988]: reset action history +[INFO] [1735524527.257435]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735524550.011769]: resetting environment +[INFO] [1735524550.013113]: cleared current plan +[INFO] [1735524550.014017]: moving to home +[INFO] [1735524551.015833]: reached home +[INFO] [1735524551.016261]: reset action history +[INFO] [1735524553.774143]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/episode_rosbags/episode_2_2024-12-29-21-09-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1357631.log +started roslaunch server http://158.130.50.37:39973/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1358169] +process[print_text-8]: started with pid [1358170] +process[rosbag_record-9]: started with pid [1358171] +started bagging! +For topic gripper_width: timestamp difference is -104993097 for nearest: 1735524579767493256 - target: 1735524579872486353 at idx 249 +[INFO] [1735524580.532470]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1735524581.302267]: Stored episode 3. +[INFO] [1735524581.308476]: Saving video... +[INFO] [1735524581.516090]: Video saved! +Starting episode 3 +[INFO] [1735524581.566316]: resetting environment +[INFO] [1735524581.570878]: cleared current plan +[INFO] [1735524581.575682]: moving to home +[INFO] [1735524585.185118]: reached home +[INFO] [1735524585.187958]: reset action history +[INFO] [1735524587.956278]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735524598.059555]: resetting environment +[INFO] [1735524598.065599]: cleared current plan +[INFO] [1735524598.068635]: moving to home +[INFO] [1735524599.073148]: reached home +[INFO] [1735524599.074978]: reset action history +[INFO] [1735524601.841083]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/episode_rosbags/episode_3_2024-12-29-21-10-01.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1357631.log +started roslaunch server http://158.130.50.37:37841/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1358322] +process[print_text-11]: started with pid [1358323] +process[rosbag_record-12]: started with pid [1358324] +started bagging! +[INFO] [1735524628.798609]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1735524629.556889]: Stored episode 4. +[INFO] [1735524629.562513]: Saving video... +[WARN] [1735524630.120123]: Plan exhausted +[WARN] [1735524630.159855]: Plan exhausted +[WARN] [1735524630.201033]: Plan exhausted +[WARN] [1735524630.240551]: Plan exhausted +[INFO] [1735524630.213842]: Video saved! +Starting episode 4 +[INFO] [1735524630.269562]: resetting environment +[INFO] [1735524630.276772]: cleared current plan +[INFO] [1735524630.285062]: moving to home +[INFO] [1735524633.909622]: reached home +[INFO] [1735524633.914685]: reset action history +[INFO] [1735524636.696429]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735524647.595735]: resetting environment +[INFO] [1735524647.596208]: cleared current plan +[INFO] [1735524647.599403]: moving to home +[INFO] [1735524648.602906]: reached home +[INFO] [1735524648.611438]: reset action history +[INFO] [1735524651.397642]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/episode_rosbags/episode_4_2024-12-29-21-10-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1357631.log +started roslaunch server http://158.130.50.37:41715/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1358452] +process[print_text-14]: started with pid [1358453] +process[rosbag_record-15]: started with pid [1358454] +started bagging! +For topic gripper_width: timestamp difference is -107005359 for nearest: 1735524659767568349 - target: 1735524659874573708 at idx 115 +For topic gripper_width: timestamp difference is -106353971 for nearest: 1735524660567538765 - target: 1735524660673892736 at idx 125 +For topic gripper_width: timestamp difference is -106191905 for nearest: 1735524660767542331 - target: 1735524660873734236 at idx 126 +For topic gripper_width: timestamp difference is -105005154 for nearest: 1735524662367501369 - target: 1735524662472506523 at idx 144 +For topic gripper_width: timestamp difference is -104814067 for nearest: 1735524662567540393 - target: 1735524662672354460 at idx 146 +For topic gripper_width: timestamp difference is -101244644 for nearest: 1735524666967499777 - target: 1735524667068744421 at idx 207 +For topic gripper_width: timestamp difference is -100697880 for nearest: 1735524667567539329 - target: 1735524667668237209 at idx 216 +For topic gripper_width: timestamp difference is -100536737 for nearest: 1735524667767540541 - target: 1735524667868077278 at idx 218 +[INFO] [1735524678.328583]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735524679.111407]: Stored episode 5. +[INFO] [1735524679.113324]: Saving video... +[INFO] [1735524679.501429]: Video saved! +[INFO] [1735524679.556492]: resetting environment +[INFO] [1735524679.564120]: cleared current plan +[INFO] [1735524679.567412]: moving to home +[INFO] [1735524683.191174]: reached home +[INFO] [1735524683.206053]: reset action history +[INFO] [1735524686.094516]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +proceeding to start episode! \ No newline at end of file diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt b/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json b/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6edb462bf411784d68d1358f917398aa210374 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-30T02:07:11.870193", + "startedAt": "2024-12-30T02:07:11.455436", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "23581857e509febdae7f0ac45cc9cf4f9081f85c" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3567.4849999999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4513.316, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4513.77, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4529.436, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 470.08051681518555 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json b/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..da26f421a252b8dabba7c0ed7ad19f834ec4330e --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "e4be4b57f794bee20f5564992ee67b022ba0fa05153f2acfd6247f5c01fe68a2", "size": 1151332, "path": "media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4"}, "global_step": 660, "_timestamp": 1735524697.9258375, "_runtime": 266.46145462989807, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "a107c4c62341f47056320339f391b4ba52de893df4146a8b852cfd2bdd2a38b6", "size": 1154425, "path": "media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "639dd4a333330f56b6d93beff55c8df46b775713592e90759c34d40aebabf5c0", "size": 1146794, "path": "media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "3be2fc636735f4996c9dcfc0866e7a227637cc528400f898e5df37430c69321b", "size": 1159865, "path": "media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "6559ba197b6fb10731c066ed282e33da3f63cd3ff933d4f145c53d0647693886", "size": 1164276, "path": "media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4"}, "_wandb": {"runtime": 277}} \ No newline at end of file diff --git a/210705/wandb/run-20241229_210711-amv7lra0/logs/debug-internal.log b/210705/wandb/run-20241229_210711-amv7lra0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..869c2cd552e7b6dbf21fb484f8c6839386d6a6c3 --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/logs/debug-internal.log @@ -0,0 +1,730 @@ +2024-12-29 21:07:11,464 INFO StreamThr :1357810 [internal.py:wandb_internal():85] W&B internal server running at pid: 1357810, started at: 2024-12-29 21:07:11.463909 +2024-12-29 21:07:11,465 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status +2024-12-29 21:07:11,479 INFO WriterThread:1357810 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb +2024-12-29 21:07:11,479 DEBUG SenderThread:1357810 [sender.py:send():379] send: header +2024-12-29 21:07:11,486 DEBUG SenderThread:1357810 [sender.py:send():379] send: run +2024-12-29 21:07:11,756 INFO SenderThread:1357810 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files +2024-12-29 21:07:11,756 INFO SenderThread:1357810 [sender.py:_start_run_threads():1188] run started: amv7lra0 with start time 1735524431.464383 +2024-12-29 21:07:11,772 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 21:07:11,772 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: check_version +2024-12-29 21:07:11,851 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 21:07:11,855 DEBUG HandlerThread:1357810 [system_info.py:__init__():26] System info init +2024-12-29 21:07:11,855 DEBUG HandlerThread:1357810 [system_info.py:__init__():41] System info init done +2024-12-29 21:07:11,855 INFO HandlerThread:1357810 [system_monitor.py:start():194] Starting system monitor +2024-12-29 21:07:11,856 INFO SystemMonitor:1357810 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 21:07:11,856 INFO HandlerThread:1357810 [system_monitor.py:probe():214] Collecting system info +2024-12-29 21:07:11,856 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 21:07:11,857 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started disk monitoring +2024-12-29 21:07:11,857 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 21:07:11,858 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started memory monitoring +2024-12-29 21:07:11,859 INFO SystemMonitor:1357810 [interfaces.py:start():188] Started network monitoring +2024-12-29 21:07:11,870 DEBUG HandlerThread:1357810 [system_info.py:probe():152] Probing system +2024-12-29 21:07:11,871 DEBUG HandlerThread:1357810 [system_info.py:_probe_git():137] Probing git +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:_probe_git():145] Probing git done +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:probe():200] Probing system done +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T02:07:11.870193', 'startedAt': '2024-12-30T02:07:11.455436', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3567.4849999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4513.316, 'min': 800.0, 'max': 4700.0}, {'current': 4513.77, 'min': 800.0, 'max': 4700.0}, {'current': 4529.436, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 470.08051681518555}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 21:07:11,876 INFO HandlerThread:1357810 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 21:07:11,876 INFO HandlerThread:1357810 [system_monitor.py:probe():227] Publishing system info +2024-12-29 21:07:11,876 DEBUG HandlerThread:1357810 [system_info.py:_save_code():44] Saving code +2024-12-29 21:07:11,877 DEBUG HandlerThread:1357810 [system_info.py:_save_code():67] Saving code done +2024-12-29 21:07:11,878 DEBUG HandlerThread:1357810 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 21:07:11,940 DEBUG HandlerThread:1357810 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 21:07:11,940 INFO HandlerThread:1357810 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 21:07:11,943 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:07:11,943 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 21:07:11,944 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 21:07:11,944 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 21:07:12,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 21:07:12,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: python_packages +2024-12-29 21:07:12,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:12,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:12,175 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:12,196 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/uvy85ybf-wandb-metadata.json +2024-12-29 21:07:12,203 INFO wandb-upload_2:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/r4am14fr-diff.patch +2024-12-29 21:07:12,212 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/3ffkt5gr-code/FISH/eval_robot.py +2024-12-29 21:07:12,217 INFO HandlerThread:1357810 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/tb" + save: true +} +control { + relay_id: "976e5e9ee00945adac42fe84b7ff337e" +} +_info { + stream_id: "amv7lra0" +} + +2024-12-29 21:07:12,217 DEBUG HandlerThread:1357810 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 21:07:12,230 DEBUG SenderThread:1357810 [sender.py:send():379] send: telemetry +2024-12-29 21:07:12,266 DEBUG SenderThread:1357810 [sender.py:send():379] send: tbrecord +2024-12-29 21:07:12,267 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:07:12,267 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 with policy live +2024-12-29 21:07:12,446 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/g38txylz-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:07:12,757 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code +2024-12-29 21:07:12,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH +2024-12-29 21:07:13,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:14,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:14,756 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:15,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:16,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:17,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:17,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:17,277 DEBUG SenderThread:1357810 [sender.py:send():379] send: config +2024-12-29 21:07:18,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:18,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:19,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:20,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:20,758 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:21,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:22,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:22,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:22,759 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:23,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:24,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:24,760 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:25,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:26,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:26,760 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:27,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:27,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:27,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:27,329 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:28,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:29,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:30,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:30,761 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:31,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:32,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:32,762 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:33,055 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:33,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:34,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:34,763 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:35,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:36,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:37,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:38,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:38,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:39,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:40,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:40,764 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:41,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:42,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:42,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:42,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:42,765 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:43,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:43,306 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:43,765 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml +2024-12-29 21:07:44,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:45,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:46,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:46,766 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:07:47,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:48,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:49,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:49,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:50,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:51,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:52,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:53,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:54,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:07:54,179 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:55,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:56,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:57,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:07:57,173 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:07:57,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:58,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:59,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:07:59,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:00,180 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:00,807 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:08:00,807 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 with policy now +2024-12-29 21:08:00,808 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:08:00,809 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:08:00,809 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:08:00,812 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:08:01,066 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/xmtm0m8l-media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:01,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:01,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media +2024-12-29 21:08:01,771 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos +2024-12-29 21:08:01,771 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:08:02,065 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/eul3dkol-media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:08:02,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:02,770 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:03,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:04,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:04,413 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:05,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:06,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:06,772 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:07,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:08,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:08,772 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:09,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:09,773 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:08:10,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:10,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:10,773 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:11,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:11,859 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 21:08:11,859 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:08:12,173 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:12,174 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:12,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:12,774 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:13,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:14,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:14,775 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:15,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:16,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:16,249 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:17,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:18,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:19,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:20,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:21,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:22,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:22,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:23,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:24,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:25,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:26,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:27,174 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:27,174 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:27,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:27,256 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:28,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:29,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:30,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:31,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:32,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:33,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:33,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:34,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:35,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:36,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:37,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:38,176 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:38,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:39,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:40,189 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:40,784 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:40,858 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:08:40,858 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 with policy now +2024-12-29 21:08:40,859 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:08:40,859 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:08:40,859 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:08:40,860 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:08:41,106 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/alfog98n-media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:41,191 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:41,784 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:08:41,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:41,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:08:41,860 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:08:42,061 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/b2tkslf7-media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:08:42,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:42,181 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:42,225 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:42,785 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:43,192 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:43,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:44,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:45,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:46,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:46,786 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:47,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:48,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:48,787 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:08:49,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:49,305 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:50,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:51,196 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:52,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:53,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:54,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:55,177 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:08:55,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:56,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:57,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:08:57,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:08:57,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:58,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:08:59,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:00,197 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:00,304 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:01,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:02,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:03,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:04,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:05,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:06,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:06,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:07,200 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:08,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:09,198 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:10,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:10,633 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:10,633 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:10,634 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:10,794 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:10,794 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:11,032 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/4pzrw60x-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:11,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:11,860 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:09:11,861 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:12,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:12,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:12,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:12,795 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:13,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:14,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:14,795 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:15,199 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:16,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:16,796 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:17,014 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:17,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:18,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:19,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:20,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:21,201 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:22,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:22,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:23,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:24,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:25,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:26,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:27,175 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:27,176 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:27,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:27,296 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:28,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:29,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:30,202 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:31,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:32,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:33,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:33,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:34,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:35,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:36,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:37,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:38,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:39,178 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:39,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:40,203 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:40,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:41,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:41,555 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:09:41,555 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 with policy now +2024-12-29 21:09:41,558 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:09:41,558 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:41,558 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:41,569 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:41,796 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/6mdf4ynb-media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:41,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:41,804 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:41,805 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:09:41,861 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:09:42,056 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/2h3qfory-media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:09:42,182 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:42,182 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:42,229 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:42,805 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:43,210 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:44,211 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:44,259 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:45,229 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:46,213 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:46,806 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:47,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:48,212 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:49,220 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:50,005 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:50,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:50,807 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:09:51,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:52,213 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:53,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:54,221 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:55,191 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:09:55,223 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:56,219 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:57,186 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:09:57,186 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:09:57,233 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:58,225 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:09:58,470 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:09:58,471 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:09:58,471 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:09:58,810 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:09:58,810 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:09:59,231 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:00,224 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:00,811 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:01,121 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:01,226 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:02,240 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:02,812 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:03,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:04,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:04,813 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:05,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:06,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:06,294 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:07,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:08,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:09,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:10,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:11,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:11,861 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:10:11,862 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:12,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:12,183 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:12,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:13,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:14,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:15,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:16,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:17,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:17,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:18,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:19,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:20,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:21,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:22,242 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:23,181 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:23,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:24,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:25,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:26,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:27,183 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:27,184 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:27,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:28,243 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:28,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:29,245 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:30,248 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:30,264 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:10:30,264 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 with policy now +2024-12-29 21:10:30,305 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:10:30,306 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:10:30,306 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:10:30,307 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:10:30,518 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/dcpthqn9-media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:30,821 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:30,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:10:31,087 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/ig4s70k7-media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:10:31,246 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:32,251 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:32,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:33,250 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:33,911 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:34,262 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:34,822 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:35,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:36,255 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:37,251 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:38,254 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:38,824 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:39,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:39,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:40,255 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:41,259 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:41,862 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:10:42,185 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:42,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:42,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:43,260 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:44,241 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:44,264 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:45,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:46,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:47,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:47,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:10:48,072 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/83t2lcan-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:10:48,262 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:48,526 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:10:48,526 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:10:48,526 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:10:48,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:10:48,827 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:49,263 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:49,645 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:50,273 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:50,828 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:51,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:52,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:52,828 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:10:53,266 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:54,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:55,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:10:55,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:56,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:57,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:10:57,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:10:57,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:58,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:10:59,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:00,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:00,715 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:00,831 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:01,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:02,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:02,832 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:03,267 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:04,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:04,833 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:05,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:05,716 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:06,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:07,268 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:08,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:08,834 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:09,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:10,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:10,916 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:11,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:11,862 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:12,184 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:12,185 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:12,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:13,269 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:14,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:15,271 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:16,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:16,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:17,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:18,272 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:19,283 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:19,553 DEBUG SenderThread:1357810 [sender.py:send():379] send: files +2024-12-29 21:11:19,553 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 with policy now +2024-12-29 21:11:19,556 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:11:19,556 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:19,556 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:19,557 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:19,803 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/m56r65qv-media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:19,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval +2024-12-29 21:11:20,087 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/si2ao02c-media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:20,298 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:20,838 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:21,288 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:21,568 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:22,300 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:23,303 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:24,300 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:24,840 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:25,306 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:26,303 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:27,137 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:27,192 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:27,193 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:27,302 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:28,327 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:28,841 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:29,308 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:30,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:31,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:32,216 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:32,307 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:33,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:34,324 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:35,313 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:36,318 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:37,217 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:37,325 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:38,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:38,585 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:38,585 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:38,585 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:38,844 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:38,844 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:11:38,845 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:39,317 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:40,320 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:41,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:41,863 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:42,190 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:11:42,190 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:11:42,304 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:42,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:43,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:44,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:45,321 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:46,323 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:47,322 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:48,219 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:48,324 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:11:48,901 DEBUG SenderThread:1357810 [sender.py:send():379] send: exit +2024-12-29 21:11:48,901 INFO SenderThread:1357810 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 21:11:48,901 INFO SenderThread:1357810 [sender.py:send_exit():588] handling runtime: 277 +2024-12-29 21:11:48,902 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:48,902 INFO SenderThread:1357810 [sender.py:send_exit():594] send defer +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 21:11:48,903 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 1 +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 21:11:48,903 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 21:11:48,903 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 2 +2024-12-29 21:11:48,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 21:11:48,903 INFO HandlerThread:1357810 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 21:11:48,903 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 21:11:48,904 DEBUG SystemMonitor:1357810 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 21:11:48,904 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 21:11:48,904 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 21:11:48,953 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [interfaces.py:finish():200] Joined network monitor +2024-12-29 21:11:48,954 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,954 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 21:11:48,954 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 3 +2024-12-29 21:11:48,954 DEBUG SenderThread:1357810 [sender.py:send():379] send: stats +2024-12-29 21:11:48,954 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,954 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 21:11:48,955 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:48,955 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 21:11:48,955 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 4 +2024-12-29 21:11:48,955 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:48,955 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 21:11:49,848 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:50,848 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:55,586 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,586 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 21:11:55,586 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 21:11:55,587 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 5 +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:55,587 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send():379] send: history +2024-12-29 21:11:55,587 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 21:11:55,587 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:55,588 DEBUG SenderThread:1357810 [sender.py:send():379] send: summary +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:11:55,588 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 21:11:55,588 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 6 +2024-12-29 21:11:55,588 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,589 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 21:11:55,589 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 7 +2024-12-29 21:11:55,589 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:11:55,589 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:55,589 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 21:11:55,589 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:55,589 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 21:11:55,850 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:11:55,902 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:58,205 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 8 +2024-12-29 21:11:58,205 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:58,205 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:58,205 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 21:11:58,206 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:58,206 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 21:11:58,206 INFO SenderThread:1357810 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 21:11:58,208 INFO SenderThread:1357810 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 21:11:58,222 INFO SenderThread:1357810 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 21:11:58,230 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 9 +2024-12-29 21:11:58,231 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:58,231 DEBUG SenderThread:1357810 [sender.py:send():379] send: artifact +2024-12-29 21:11:58,231 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 21:11:58,685 INFO wandb-upload_2:1357810 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8w1_xu6h +2024-12-29 21:11:58,851 INFO Thread-12 :1357810 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:11:58,865 INFO wandb-upload_0:1357810 [upload_job.py:push():88] Uploaded file /tmp/tmpg1hqer99/wandb-job.json +2024-12-29 21:11:58,867 INFO wandb-upload_1:1357810 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4rwvgu1e +2024-12-29 21:11:58,903 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA2MzIwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA1NjUzOQ==', 'versionIndex': 386}}} +2024-12-29 21:11:59,443 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 21:11:59,443 INFO SenderThread:1357810 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 21:11:59,851 INFO SenderThread:1357810 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml config.yaml +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json wandb-summary.json +2024-12-29 21:11:59,852 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt requirements.txt +2024-12-29 21:11:59,855 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-metadata.json wandb-metadata.json +2024-12-29 21:11:59,855 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:11:59,856 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log output.log +2024-12-29 21:11:59,856 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/diff.patch diff.patch +2024-12-29 21:11:59,857 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 21:11:59,857 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 media/videos/eval/1_eval_1_a107c4c62341f4705632.mp4 +2024-12-29 21:11:59,859 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 media/videos/eval/0_eval_0_e4be4b57f794bee20f55.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 media/videos/eval/3_eval_3_3be2fc636735f4996c9d.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 media/videos/eval/4_eval_4_6559ba197b6fb10731c0.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 media/videos/eval/2_eval_2_639dd4a333330f56b6d9.mp4 +2024-12-29 21:11:59,860 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 10 +2024-12-29 21:11:59,860 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:11:59,861 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:11:59,861 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 21:11:59,861 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:11:59,861 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 21:11:59,861 INFO SenderThread:1357810 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:11:59,904 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:11:59,905 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,027 INFO wandb-upload_0:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/wandb-summary.json +2024-12-29 21:12:00,027 INFO wandb-upload_2:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/config.yaml +2024-12-29 21:12:00,061 INFO wandb-upload_1:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/requirements.txt +2024-12-29 21:12:00,133 INFO wandb-upload_4:1357810 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/files/output.log +2024-12-29 21:12:00,164 INFO wandb-upload_3:1357810 [upload_job.py:push():130] Uploaded file /tmp/tmpzik4_qmkwandb/2wpcfjl2-events.out.tfevents.1735524432.leonmkim-ROG-Strix-G15CS-G15CS.1357631.0 +2024-12-29 21:12:00,364 INFO Thread-11 (_thread_body):1357810 [sender.py:transition_state():614] send defer: 11 +2024-12-29 21:12:00,364 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,364 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 21:12:00,365 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,365 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 21:12:00,365 INFO SenderThread:1357810 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 12 +2024-12-29 21:12:00,366 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,366 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 21:12:00,366 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 21:12:00,366 INFO SenderThread:1357810 [file_stream.py:finish():601] file stream finish called +2024-12-29 21:12:00,616 INFO SenderThread:1357810 [file_stream.py:finish():605] file stream finish is done +2024-12-29 21:12:00,616 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 13 +2024-12-29 21:12:00,616 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,616 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 21:12:00,616 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:transition_state():614] send defer: 14 +2024-12-29 21:12:00,617 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:12:00,617 INFO HandlerThread:1357810 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send():379] send: final +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send():379] send: footer +2024-12-29 21:12:00,617 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: defer +2024-12-29 21:12:00,617 INFO SenderThread:1357810 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 21:12:00,617 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:12:00,618 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,618 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 21:12:00,618 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: server_info +2024-12-29 21:12:00,619 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 21:12:00,620 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:12:00,716 DEBUG SenderThread:1357810 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 21:12:00,716 INFO MainThread:1357810 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 21:12:00,717 DEBUG HandlerThread:1357810 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 21:12:00,717 INFO HandlerThread:1357810 [handler.py:finish():882] shutting down handler +2024-12-29 21:12:01,620 INFO WriterThread:1357810 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [sender.py:finish():1615] shutting down sender +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:12:01,716 INFO SenderThread:1357810 [file_pusher.py:join():175] waiting for file pusher diff --git a/210705/wandb/run-20241229_210711-amv7lra0/logs/debug.log b/210705/wandb/run-20241229_210711-amv7lra0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6e4c547498baf6eff48c34c93d3c4890e4d6d64f --- /dev/null +++ b/210705/wandb/run-20241229_210711-amv7lra0/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-29 21:07:11,457 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Configure stats pid to 1357631 +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/settings +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/logs/debug.log +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/wandb/run-20241229_210711-amv7lra0/logs/debug-internal.log +2024-12-29 21:07:11,458 INFO MainThread:1357631 [wandb_init.py:init():569] calling init triggers +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 5, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1051_0', 'wandb_notes': '1051_0_req_1368_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705'} +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():619] starting backend +2024-12-29 21:07:11,459 INFO MainThread:1357631 [wandb_init.py:init():623] setting up manager +2024-12-29 21:07:11,463 INFO MainThread:1357631 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 21:07:11,464 INFO MainThread:1357631 [wandb_init.py:init():631] backend started and connected +2024-12-29 21:07:11,478 INFO MainThread:1357631 [wandb_init.py:init():720] updated telemetry +2024-12-29 21:07:11,484 INFO MainThread:1357631 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 21:07:11,772 INFO MainThread:1357631 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 21:07:11,846 INFO MainThread:1357631 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 21:07:11,846 INFO MainThread:1357631 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 21:07:12,173 INFO MainThread:1357631 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 21:07:12,174 INFO MainThread:1357631 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 21:07:12,176 INFO MainThread:1357631 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 21:07:12,177 INFO MainThread:1357631 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1051_0/210705/tb, True +2024-12-29 21:07:17,276 INFO MainThread:1357631 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 21:12:01,717 WARNING MsgRouterThr:1357631 [router.py:message_loop():77] message_loop has been closed diff --git a/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb b/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..42706f1ccea706dad79420f374ca75cf414db4f3 Binary files /dev/null and b/210705/wandb/run-20241229_210711-amv7lra0/run-amv7lra0.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e4069b2e73c21850fe99741529c212a95fa87b --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 5 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1051_0' +wandb_notes: 1051_0_req_1368_0 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 0 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..609516576823d3171a60afb3553005f19867a47b --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eee2d8be44037703625338c5a61318ccf53ead7833a817363cd3952995728a78 +size 910065238 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cf25df67c457242a9504edf62974acd813dc479 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b564eb2c3ac6481d4155f45b4b2fd53fe4ecd2a6705275cea52f09abb195080 +size 910065238 diff --git a/snapshot_13249.pt b/snapshot_13249.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c5e21f5b5a6e12b6033436d7ee600e4bfeabbb0 --- /dev/null +++ b/snapshot_13249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40daf299e13601efb1f688ee54433b399ca192028202ce502688e2ccb961c652 +size 910065238 diff --git a/snapshot_13499.pt b/snapshot_13499.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfb76787b0de4c33f36650d2ff5f97f8e43be4e0 --- /dev/null +++ b/snapshot_13499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4c8912dad4aeb9185f3727863c16fcf03b4ce9c61931305ff57631d6065c95 +size 910065238 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2da26b4ca3aff9602bd5d3ce6cbd28b844bf40f --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae56699b30855d0d5dae4de33db9b3448268fda2371fb981015e1e4daa83dff2 +size 910065238 diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff842a2f7097b85ec49ed39691267026bfaba259 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a8f9ba18495e1e33ba4dc839e6f27185b61fd48c008f7f3346a0ba3fbef33be +size 910065238 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae998673a19b4e69afba8613ef710f5354698134 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:018a117b342238f460fb92603c88676a0727043e80593daadf59c76e1cef15b0 +size 910065238 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..aeeacacf369c384e74e21d57961cd42ee91e783c --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07ab78b0142288959d0c8eb19b98e4ce1e6c3dd2ee72a10b60ae4abe2716e56 +size 910065238 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..c55d1a8e9a8274a69e04262a795c2cc600c08f14 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f9d8c3f172e49e8f38475109dcad601b73111c3fae1677e9be4eee8f0bc95c +size 910065238 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..25f98b2276dacb010b2aaeef65b70b207e84381f --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60ecbb94d290a7dd552a92707bebe2d3fd2fbeef9d456014d782dbb5c6addf56 +size 910065238 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b781b053a6be883b4a510f8adb5014efd001ea16 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef0bbe3ee5f39e21fba3a8766a74911da143b61fb145e00f3bff6cec50a5edd0 +size 910065238 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f1e9fc0239ab7a3aa9b880be6980b7364373b4d --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b8ba6d3391c0b8a8d75546512b57892e1dde216047177b3332fb8b54eeb871d +size 910065238 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..01daeaf0c8ff8821ce17352d547baf05303ce063 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a42eab4150af49aa5a74939ae39cfd2bb44a68327c78d6469aa4d7070c6e92b +size 910065238 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..c55d1a8e9a8274a69e04262a795c2cc600c08f14 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f9d8c3f172e49e8f38475109dcad601b73111c3fae1677e9be4eee8f0bc95c +size 910065238