diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..de85294a912a34a2975271c034fdfc6015aef441 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,42 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nibxxtai/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kdczwpup/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +p7zqzp4h/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +nbjbj4mc/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5425b89195ed5aef3e2666dd0c5ae90d41fdcd73 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 1 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1930_0' +wandb_notes: 1930_0_restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/kdczwpup/.hydra/config.yaml b/kdczwpup/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/kdczwpup/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kdczwpup/.hydra/hydra.yaml b/kdczwpup/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b36d303ad190578c460182597bf44a008f7a5e6e --- /dev/null +++ b/kdczwpup/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kdczwpup/.hydra/overrides.yaml b/kdczwpup/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kdczwpup/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kdczwpup/episode_rosbags/aligned_depth_to_color_K.npy b/kdczwpup/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kdczwpup/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kdczwpup/episode_rosbags/cam_tf_world.npy b/kdczwpup/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kdczwpup/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kdczwpup/episode_rosbags/color_K.npy b/kdczwpup/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kdczwpup/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kdczwpup/episode_rosbags/depth_K.npy b/kdczwpup/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kdczwpup/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kdczwpup/eval_robot.log b/kdczwpup/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ce30abbe20027b30d8c8176ba13f91e92ec2041b --- /dev/null +++ b/kdczwpup/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:52:30,439][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:52:30,442][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kdczwpup/eval_video/0_eval.mp4 b/kdczwpup/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef4e24d6f7f21aa657b98c0c8cbc750ca655b29a --- /dev/null +++ b/kdczwpup/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c48f041424ff047394474d7a401a215b0f1a4be94a4db51735004355ca95c53 +size 1438154 diff --git a/kdczwpup/eval_video/1_eval.mp4 b/kdczwpup/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d5288c1ae60682b4e03c0a83a21d916bdffad83 --- /dev/null +++ b/kdczwpup/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6117da8fb30ba309d25209833f1f9a58981cf0bc96656d8114f6c4d1a5148d14 +size 1447561 diff --git a/kdczwpup/eval_video/2_eval.mp4 b/kdczwpup/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b86465d54478900c2b230de835067fe8f786ecec --- /dev/null +++ b/kdczwpup/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a554ab7142f253de1eefcf0997fa0677afc7503234c9af7b5695dcad9295c08c +size 1451623 diff --git a/kdczwpup/eval_video/3_eval.mp4 b/kdczwpup/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11ecdc7aa77102f2ee25196744ba300271f9f618 --- /dev/null +++ b/kdczwpup/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d248f3bb3a8cadb810ff9d8179f9946a878b0b3d1307fdd347f1c7d79bee5bf +size 1468793 diff --git a/kdczwpup/eval_video/4_eval.mp4 b/kdczwpup/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b178258481ffb2b2809537ab4b7d98d8a50e274b --- /dev/null +++ b/kdczwpup/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42b418c78bb7f1b89703206b083bfc4242af9452b6a006664e2852861650436b +size 1435179 diff --git a/kdczwpup/tb/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 b/kdczwpup/tb/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 new file mode 100644 index 0000000000000000000000000000000000000000..fa0aa83310743ccbc0039e00103a13df1823d280 --- /dev/null +++ b/kdczwpup/tb/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f7d32ee82133d63d603a511626702a795f2f296dd4dc54f00be5b9976ee96e9 +size 6539 diff --git a/kdczwpup/wandb/debug-internal.log b/kdczwpup/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1473c73fa36877b7f120aa6ad62809174819dcd4 --- /dev/null +++ b/kdczwpup/wandb/debug-internal.log @@ -0,0 +1,795 @@ +2025-01-03 20:52:36,185 INFO StreamThr :1970913 [internal.py:wandb_internal():85] W&B internal server running at pid: 1970913, started at: 2025-01-03 20:52:36.185175 +2025-01-03 20:52:36,186 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:52:36,198 INFO WriterThread:1970913 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb +2025-01-03 20:52:36,198 DEBUG SenderThread:1970913 [sender.py:send():379] send: header +2025-01-03 20:52:36,207 DEBUG SenderThread:1970913 [sender.py:send():379] send: run +2025-01-03 20:52:36,436 INFO SenderThread:1970913 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files +2025-01-03 20:52:36,436 INFO SenderThread:1970913 [sender.py:_start_run_threads():1188] run started: kdczwpup with start time 1735955556.186292 +2025-01-03 20:52:36,453 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:52:36,454 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:52:36,537 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:52:36,542 DEBUG HandlerThread:1970913 [system_info.py:__init__():26] System info init +2025-01-03 20:52:36,542 DEBUG HandlerThread:1970913 [system_info.py:__init__():41] System info init done +2025-01-03 20:52:36,542 INFO HandlerThread:1970913 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:52:36,542 INFO SystemMonitor:1970913 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:52:36,542 INFO HandlerThread:1970913 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:52:36,543 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:52:36,543 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:52:36,544 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:52:36,544 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:52:36,546 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:52:36,562 DEBUG HandlerThread:1970913 [system_info.py:probe():152] Probing system +2025-01-03 20:52:36,563 DEBUG HandlerThread:1970913 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:probe():200] Probing system done +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:52:36.562203', 'startedAt': '2025-01-04T01:52:36.175252', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.2647500000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.229, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 770.2987747192383}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:52:36,568 INFO HandlerThread:1970913 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:52:36,568 INFO HandlerThread:1970913 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:_save_code():44] Saving code +2025-01-03 20:52:36,570 DEBUG HandlerThread:1970913 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:52:36,570 DEBUG HandlerThread:1970913 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:52:36,633 DEBUG HandlerThread:1970913 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:52:36,633 INFO HandlerThread:1970913 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:52:36,636 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:52:36,636 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:52:36,636 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:52:36,637 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:52:36,887 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:52:36,887 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:52:36,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:36,888 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/vml0rvyr-wandb-metadata.json +2025-01-03 20:52:36,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:52:36,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:52:36,900 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/k09ugcyl-code/FISH/eval_robot.py +2025-01-03 20:52:36,933 INFO HandlerThread:1970913 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/tb" + save: true +} +control { + relay_id: "ab8a610a85414abebe45349ef347c6c8" +} +_info { + stream_id: "kdczwpup" +} + +2025-01-03 20:52:36,933 DEBUG HandlerThread:1970913 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:52:36,946 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/a2nxh3us-diff.patch +2025-01-03 20:52:37,000 DEBUG SenderThread:1970913 [sender.py:send():379] send: telemetry +2025-01-03 20:52:37,000 DEBUG SenderThread:1970913 [sender.py:send():379] send: tbrecord +2025-01-03 20:52:37,001 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:52:37,001 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 with policy live +2025-01-03 20:52:37,208 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/j1es67v0-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code +2025-01-03 20:52:37,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:38,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:39,437 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:39,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:40,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:41,518 DEBUG SenderThread:1970913 [sender.py:send():379] send: config +2025-01-03 20:52:41,518 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:41,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:42,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:43,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:43,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:44,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:45,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:45,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:46,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:46,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:47,440 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:47,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:48,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:49,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:50,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:51,441 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:51,887 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:52:51,888 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:52:51,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:52,003 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:52,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:53,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:54,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:55,442 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:55,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:56,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:57,443 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:57,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:57,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:58,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:59,444 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:59,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:00,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:01,444 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:01,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:02,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:02,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:03,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:04,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:05,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:06,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:06,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:06,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:07,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:08,015 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:08,446 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:53:08,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:09,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:10,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:11,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:12,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:13,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:13,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:14,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:15,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:16,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:17,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:18,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:18,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:19,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:20,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:21,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:21,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:21,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:22,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:23,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:24,701 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:24,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:25,451 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:25,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:26,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:27,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:28,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:29,452 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:29,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:29,989 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:30,899 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:31,453 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:31,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:32,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:33,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:34,105 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:53:34,106 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 with policy now +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:53:34,107 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:53:34,380 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/zd9xeabn-media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos +2025-01-03 20:53:34,697 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/2cued4j5-media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:35,108 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:35,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:35,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:36,546 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:53:36,546 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:53:36,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:36,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:37,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:38,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:39,455 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:39,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:40,408 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:40,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:41,456 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:41,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:42,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:43,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:44,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:45,485 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:45,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:46,458 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:53:46,704 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/kq5mzfdj-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:53:46,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:47,458 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:47,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:48,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:49,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:50,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:51,460 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:51,466 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:51,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:51,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:52,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:53,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:54,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:55,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:56,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:56,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:57,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:58,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:59,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:00,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:01,463 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:01,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:01,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:02,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:03,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:04,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:05,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:06,554 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:54:06,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:06,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:06,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:06,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:07,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:08,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:09,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:10,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:11,466 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:11,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:11,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:12,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:13,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:14,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:15,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:16,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:16,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:17,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:18,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:19,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:20,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:21,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:21,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:21,891 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:21,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:22,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:23,470 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:23,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:24,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:25,181 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:54:25,181 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 with policy now +2025-01-03 20:54:25,184 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:54:25,184 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:54:25,184 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:54:25,186 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:54:25,430 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/9zwx3vd7-media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:54:25,732 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/5f147apn-media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,908 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:26,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:26,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:27,472 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:27,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:28,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:29,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:30,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:31,473 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:31,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:32,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:32,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:33,473 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:33,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:34,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:54:34,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:35,347 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:54:35,347 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:54:35,348 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:54:35,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:54:35,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:35,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:36,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:54:36,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:36,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:37,475 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:37,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:37,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:38,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:39,476 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:39,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:40,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:41,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:42,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:43,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:43,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:44,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:45,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:46,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:47,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:48,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:49,093 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:49,479 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:49,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:50,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:51,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:51,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:52,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:53,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:54,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:55,049 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:55,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:56,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:57,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:58,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:59,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:00,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:00,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:01,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:02,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:03,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:04,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:05,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:05,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:06,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:55:06,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:06,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:06,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:07,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:08,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:09,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:10,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:11,008 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:11,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:12,957 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:13,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:13,958 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:55:13,958 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 with policy now +2025-01-03 20:55:14,001 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:55:14,002 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:55:14,002 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:55:14,004 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:55:14,208 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/jx1qz91x-media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,487 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:55:14,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:55:14,813 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/zlwjal09-media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,921 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:15,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:15,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:16,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:16,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:17,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:18,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:19,489 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:19,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:20,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:21,490 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:21,893 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:21,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:21,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:21,994 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:22,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:23,924 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:24,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:25,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:26,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:27,899 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:27,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:28,924 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:29,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:30,196 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:55:30,196 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:55:30,197 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:55:30,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:55:30,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:55:30,737 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/votd4cre-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:55:30,925 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:31,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:31,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:32,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:33,494 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:33,510 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:33,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:34,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:35,495 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:35,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:36,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:55:36,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:36,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:37,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:38,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:39,003 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:39,928 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:40,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:41,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:42,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:43,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:44,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:44,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:45,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:46,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:47,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:48,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:49,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:49,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:50,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:51,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:51,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:52,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:53,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:54,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:55,029 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:55,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:56,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:57,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:58,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:59,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:00,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:00,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:01,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:02,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:03,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:04,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:05,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:05,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:06,549 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:56:06,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:06,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:06,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:07,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:08,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:09,938 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:10,678 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:56:10,679 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 with policy now +2025-01-03 20:56:10,682 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:56:10,682 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:56:10,682 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:56:10,684 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:56:10,939 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:10,941 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/99skfep5-media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:56:11,735 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:11,851 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/lkbgp57w-media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,942 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:12,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:13,510 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:13,943 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:14,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:15,511 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:15,952 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:16,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:16,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:17,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:18,944 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:19,512 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:19,944 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:20,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:21,258 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:56:21,258 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:56:21,258 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:56:21,513 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:56:21,513 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:56:21,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:21,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:21,913 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:21,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:22,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:23,514 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:23,947 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:24,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:25,514 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:25,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:26,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:27,515 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:27,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:27,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:28,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:29,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:30,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:31,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:32,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:33,469 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:33,517 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:33,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:34,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:35,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:36,550 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:56:36,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:36,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:36,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:37,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:38,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:39,055 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:39,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:40,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:41,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:42,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:43,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:44,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:44,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:45,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:46,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:47,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:48,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:49,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:49,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:50,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:51,893 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:51,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:51,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:52,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:53,525 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:53,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:54,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:55,477 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:55,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:56,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:57,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:58,951 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:59,953 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:00,523 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:00,583 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:00,583 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 with policy now +2025-01-03 20:57:00,586 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:00,586 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:00,586 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:00,593 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:00,855 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/pw2k4bo7-media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:00,956 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:57:01,823 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/axefb25s-media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:01,960 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:02,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:03,959 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:04,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:05,530 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:05,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:06,385 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:06,550 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:57:06,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:57:06,904 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:57:06,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:07,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:08,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:09,532 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:09,964 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:10,966 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:11,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:11,973 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:12,985 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:13,976 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:14,121 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:57:14,121 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval +2025-01-03 20:57:14,593 INFO wandb-upload_2:1970913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr0euz6my +2025-01-03 20:57:14,775 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/yzyp9d7i-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:14,784 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/w32wdhiq-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:14,966 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:15,001 INFO SenderThread:1970913 [sender.py:send_request_log_artifact():1525] logged artifact run-kdczwpup-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3ODMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NTE2NA==', 'latestArtifact': None}} +2025-01-03 20:57:15,001 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:15,001 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:57:15,002 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:15,002 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:15,002 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:15,242 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/8f5hsqst-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:15,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:15,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:15,968 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:16,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:17,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:17,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:18,815 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:18,815 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:18,815 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:18,816 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:18,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:19,080 DEBUG SenderThread:1970913 [sender.py:send():379] send: telemetry +2025-01-03 20:57:19,124 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,124 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,124 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,125 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,125 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,125 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,125 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,126 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,126 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,126 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,126 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,127 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,127 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,127 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,127 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,128 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,128 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,128 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,128 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,129 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,129 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,129 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,184 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:57:19,184 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:57:19,195 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval +2025-01-03 20:57:19,583 INFO wandb-upload_0:1970913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp57pjjwrm +2025-01-03 20:57:19,746 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/iyrqw4jc-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:19,928 INFO SenderThread:1970913 [sender.py:send_request_log_artifact():1525] logged artifact run-kdczwpup-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3ODQxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NTE3OQ==', 'latestArtifact': None}} +2025-01-03 20:57:19,928 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:19,928 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json with policy now +2025-01-03 20:57:19,929 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:19,929 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,929 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:20,157 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/12xjymbx-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:20,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:20,968 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:21,237 DEBUG SenderThread:1970913 [sender.py:send():379] send: exit +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():588] handling runtime: 284 +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():594] send defer +2025-01-03 20:57:21,238 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,238 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:57:21,238 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:57:21,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:57:21,239 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:57:21,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:57:21,240 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:57:21,240 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:57:21,240 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:57:21,292 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:57:21,293 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:57:21,293 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:57:21,293 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,293 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:57:21,293 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:57:21,293 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:57:21,294 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,294 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:57:21,294 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,294 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:57:21,294 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:57:21,294 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,294 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:57:21,537 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:21,537 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:23,538 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,333 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:57:27,333 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:27,333 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,333 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:27,334 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:27,335 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:27,335 DEBUG SenderThread:1970913 [sender.py:send():379] send: summary +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:27,339 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:57:27,339 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,339 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:57:27,340 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,340 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:57:27,343 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:27,462 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:57:27,462 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,462 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:57:27,462 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,462 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:57:27,539 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:27,539 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:57:28,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:30,753 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:57:30,753 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:30,753 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:30,753 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:57:30,754 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:30,754 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:57:30,754 INFO SenderThread:1970913 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:57:30,756 INFO SenderThread:1970913 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:57:30,758 INFO SenderThread:1970913 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:57:30,770 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:57:30,770 DEBUG SenderThread:1970913 [sender.py:send():379] send: artifact +2025-01-03 20:57:30,770 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:30,773 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 20:57:30,967 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:57:31,240 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:31,540 INFO SenderThread:1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml config.yaml +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json wandb-summary.json +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt requirements.txt +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log output.log +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch diff.patch +2025-01-03 20:57:31,542 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:57:31,550 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:31,551 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:31,551 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:57:31,551 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:31,551 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:57:31,551 INFO SenderThread:1970913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:57:31,726 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt +2025-01-03 20:57:31,734 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:31,744 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:57:31,778 INFO wandb-upload_4:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:31,843 INFO wandb-upload_3:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/l8txmwyn-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:32,043 INFO Thread-11 (_thread_body):1970913 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:57:32,043 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,043 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:57:32,044 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,044 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:57:32,044 INFO SenderThread:1970913 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:57:32,045 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:57:32,045 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,045 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:57:32,045 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,045 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:57:32,046 INFO SenderThread:1970913 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:57:32,226 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,226 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:57:32,226 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:57:32,226 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,226 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send():379] send: final +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send():379] send: footer +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,227 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:57:32,227 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:32,228 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:57:32,230 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:57:32,230 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:32,289 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:32,289 INFO MainThread:1970913 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:57:32,290 INFO MainThread:1970913 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:57:32,290 INFO MainThread:1970913 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:57:32,290 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:57:32,290 INFO HandlerThread:1970913 [handler.py:finish():882] shutting down handler +2025-01-03 20:57:33,228 INFO WriterThread:1970913 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [sender.py:finish():1615] shutting down sender +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [file_pusher.py:join():175] waiting for file pusher diff --git a/kdczwpup/wandb/debug.log b/kdczwpup/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c70f35b2bdaaf2339c7947dd4d87bcbaeb76ee42 --- /dev/null +++ b/kdczwpup/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Configure stats pid to 1970738 +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/settings +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug.log +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug-internal.log +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup'} +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():619] starting backend +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():623] setting up manager +2025-01-03 20:52:36,184 INFO MainThread:1970738 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:52:36,186 INFO MainThread:1970738 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:52:36,198 INFO MainThread:1970738 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:52:36,204 INFO MainThread:1970738 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:52:36,453 INFO MainThread:1970738 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:52:36,491 INFO MainThread:1970738 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:52:36,492 INFO MainThread:1970738 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:52:36,890 INFO MainThread:1970738 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:52:36,890 INFO MainThread:1970738 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/tb, True +2025-01-03 20:52:41,518 INFO MainThread:1970738 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:57:33,291 WARNING MsgRouterThr:1970738 [router.py:message_loop():77] message_loop has been closed diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..374a984df8d2267f938149123d9550dfa2219a28 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735955556 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..260fed1aee926a75f7c67b83fb8ec5b323d21693 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..703ec3d 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cce0f428f851722b3cb84f2174378d6996531209 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[10, 10, 1.0, 0.6597032546677664, 1, 0.7793639599718221, 1, 0.8272221683801428, 1, 0.8133141546814853, 1]]} \ No newline at end of file diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ef4e24d6f7f21aa657b98c0c8cbc750ca655b29a --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c48f041424ff047394474d7a401a215b0f1a4be94a4db51735004355ca95c53 +size 1438154 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d5288c1ae60682b4e03c0a83a21d916bdffad83 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6117da8fb30ba309d25209833f1f9a58981cf0bc96656d8114f6c4d1a5148d14 +size 1447561 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b86465d54478900c2b230de835067fe8f786ecec --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a554ab7142f253de1eefcf0997fa0677afc7503234c9af7b5695dcad9295c08c +size 1451623 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11ecdc7aa77102f2ee25196744ba300271f9f618 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d248f3bb3a8cadb810ff9d8179f9946a878b0b3d1307fdd347f1c7d79bee5bf +size 1468793 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b178258481ffb2b2809537ab4b7d98d8a50e274b --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42b418c78bb7f1b89703206b083bfc4242af9452b6a006664e2852861650436b +size 1435179 diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a6b025c266fa4185deeaebb370bae658cc2c587d --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log @@ -0,0 +1,277 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735955561.519152]: resetting environment +[INFO] [1735955561.523637]: cleared current plan +[INFO] [1735955561.524668]: moving to home +[INFO] [1735955562.526825]: reached home +[INFO] [1735955562.528121]: reset action history +[INFO] [1735955565.240225]: environment reset +Starting episode 0 +[INFO] [1735955565.244714]: resetting environment +[INFO] [1735955565.250206]: cleared current plan +[INFO] [1735955565.251331]: moving to home +[INFO] [1735955566.253634]: reached home +[INFO] [1735955566.254951]: reset action history +[INFO] [1735955568.964006]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735955574.350240]: resetting environment +[INFO] [1735955574.354595]: cleared current plan +[INFO] [1735955574.355544]: moving to home +[INFO] [1735955575.357267]: reached home +[INFO] [1735955575.359034]: reset action history +[INFO] [1735955578.068584]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/episode_rosbags/episode_0_2025-01-03-20-52-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1970738.log +started roslaunch server http://158.130.50.37:35243/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1971018] +process[print_text-2]: started with pid [1971042] +process[rosbag_record-3]: started with pid [1971043] +started bagging! +For topic gripper_width: timestamp difference is -184468063 for nearest: 1735955579461594788 - target: 1735955579646062851 at idx 19 +For topic gripper_width: timestamp difference is 103116942 for nearest: 1735955602628260565 - target: 1735955602525143623 at idx 249 +For topic gripper_width: timestamp difference is 106888588 for nearest: 1735955607328257378 - target: 1735955607221368790 at idx 249 +For topic gripper_width: timestamp difference is 108226021 for nearest: 1735955608928259333 - target: 1735955608820033312 at idx 249 +For topic gripper_width: timestamp difference is 109576634 for nearest: 1735955610528310469 - target: 1735955610418733835 at idx 249 +[INFO] [1735955613.127598]: Storing episode... +[pub_text-1] killing on exit +[print_text-2] killing on exit[rosbag_record-3] killing on exit +[INFO] [1735955613.846878]: Stored episode 1. +[INFO] [1735955613.847132]: Saving video... +[INFO] [1735955614.057134]: Video saved! +Starting episode 1 +[INFO] [1735955614.078372]: resetting environment +[INFO] [1735955614.089362]: cleared current plan +[INFO] [1735955614.089644]: moving to home +[INFO] [1735955617.692905]: reached home +[INFO] [1735955617.696389]: reset action history +[INFO] [1735955620.408215]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955625.491822]: resetting environment +[INFO] [1735955625.498184]: cleared current plan +[INFO] [1735955625.498667]: moving to home +[INFO] [1735955626.500808]: reached home +[INFO] [1735955626.501433]: reset action history +[INFO] [1735955629.210346]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/episode_rosbags/episode_1_2025-01-03-20-53-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1970738.log +started roslaunch server http://158.130.50.37:46437/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1971166] +process[print_text-5]: started with pid [1971167] +process[rosbag_record-6]: started with pid [1971191] +started bagging! +For topic gripper_width: timestamp difference is 101138201 for nearest: 1735955639828259077 - target: 1735955639727120876 at idx 137 +For topic gripper_width: timestamp difference is 108622573 for nearest: 1735955650561595224 - target: 1735955650452972651 at idx 249 +For topic gripper_width: timestamp difference is 117105644 for nearest: 1735955661228256386 - target: 1735955661111150742 at idx 249 +[INFO] [1735955664.075290]: Storing episode... +[pub_text-4] killing on exit +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[INFO] [1735955664.856688]: Stored episode 2. +[INFO] [1735955664.856989]: Saving video... +[INFO] [1735955665.140591]: Video saved! +Starting episode 2 +[INFO] [1735955665.185761]: resetting environment +[INFO] [1735955665.213667]: cleared current plan +[INFO] [1735955665.213921]: moving to home +[INFO] [1735955668.918707]: reached home +[INFO] [1735955668.922655]: reset action history +[INFO] [1735955671.649250]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955674.156291]: resetting environment +[INFO] [1735955674.162702]: cleared current plan +[INFO] [1735955674.162873]: moving to home +[INFO] [1735955675.164301]: reached home +[INFO] [1735955675.165118]: reset action history +[INFO] [1735955677.876868]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/episode_rosbags/episode_2_2025-01-03-20-54-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1970738.log +started roslaunch server http://158.130.50.37:45333/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1971333] +process[print_text-8]: started with pid [1971334] +process[rosbag_record-9]: started with pid [1971335] +started bagging! +For topic gripper_width: timestamp difference is 104729087 for nearest: 1735955687028275163 - target: 1735955686923546076 at idx 125 +[INFO] [1735955712.915802]: Storing episode... +[rosbag_record-9] killing on exit[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735955713.691997]: Stored episode 3. +[INFO] [1735955713.694343]: Saving video... +[INFO] [1735955713.910079]: Video saved! +Starting episode 3 +[INFO] [1735955713.965276]: resetting environment +[INFO] [1735955713.983396]: cleared current plan +[INFO] [1735955713.984905]: moving to home +[INFO] [1735955717.591072]: reached home +[INFO] [1735955717.600083]: reset action history +[INFO] [1735955720.326223]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955729.702213]: resetting environment +[INFO] [1735955729.720127]: cleared current plan +[INFO] [1735955729.723190]: moving to home +[INFO] [1735955730.728062]: reached home +[INFO] [1735955730.730414]: reset action history +[INFO] [1735955733.508768]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/episode_rosbags/episode_3_2025-01-03-20-55-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1970738.log +started roslaunch server http://158.130.50.37:40965/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1971477] +process[print_text-11]: started with pid [1971478] +process[rosbag_record-12]: started with pid [1971479] +started bagging! +For topic gripper_width: timestamp difference is -258623954 for nearest: 1735955768761626128 - target: 1735955769020250082 at idx 249 +[INFO] [1735955769.349237]: Storing episode... +[print_text-11] killing on exit[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1735955770.101049]: Stored episode 4. +[INFO] [1735955770.103598]: Saving video... +[INFO] [1735955770.596969]: Video saved! +Starting episode 4 +[INFO] [1735955770.683609]: resetting environment +[INFO] [1735955770.728224]: cleared current plan +[INFO] [1735955770.734641]: moving to home +[WARN] [1735955770.677717]: Plan exhausted +[WARN] [1735955770.717675]: Plan exhausted +[INFO] [1735955774.438966]: reached home +[INFO] [1735955774.442050]: reset action history +[INFO] [1735955777.239220]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955780.843102]: resetting environment +[INFO] [1735955780.867018]: cleared current plan +[INFO] [1735955780.868900]: moving to home +[INFO] [1735955781.872822]: reached home +[INFO] [1735955781.874509]: reset action history +[INFO] [1735955784.616087]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/episode_rosbags/episode_4_2025-01-03-20-56-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1970738.log +started roslaunch server http://158.130.50.37:36073/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1971596] +process[print_text-14]: started with pid [1971597] +process[rosbag_record-15]: started with pid [1971621] +started bagging! +For topic gripper_width: timestamp difference is 118298710 for nearest: 1735955786161628426 - target: 1735955786043329716 at idx 21 +For topic gripper_width: timestamp difference is 118466276 for nearest: 1735955786428258795 - target: 1735955786309792519 at idx 23 +For topic gripper_width: timestamp difference is 113530996 for nearest: 1735955792428289535 - target: 1735955792314758539 at idx 94 +For topic gripper_width: timestamp difference is 102064651 for nearest: 1735955811628285688 - target: 1735955811526221037 at idx 249 +For topic gripper_width: timestamp difference is 103743577 for nearest: 1735955812428289199 - target: 1735955812324545622 at idx 249 +[INFO] [1735955819.419889]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1735955820.215229]: Stored episode 5. +[INFO] [1735955820.224723]: Saving video... +[INFO] [1735955820.523427]: Video saved! +[INFO] [1735955820.589055]: resetting environment +[INFO] [1735955820.632382]: cleared current plan +[INFO] [1735955820.633985]: moving to home +[INFO] [1735955824.340448]: reached home +[INFO] [1735955824.342466]: reset action history +[INFO] [1735955827.144471]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a9184ca5db5ef3a0d7f000bc2ba37e89aca950 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:52:36.562203", + "startedAt": "2025-01-04T01:52:36.175252", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.2647500000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4498.229, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 770.2987747192383 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..211a24a9954622f08d6c34a6509a44b3356da944 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "9c48f041424ff047394474d7a401a215b0f1a4be94a4db51735004355ca95c53", "size": 1438154, "path": "media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4"}, "global_step": 665, "_timestamp": 1735955839.0940263, "_runtime": 282.9077343940735, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6117da8fb30ba309d25209833f1f9a58981cf0bc96656d8114f6c4d1a5148d14", "size": 1447561, "path": "media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4"}, "eval/num_success": 5.0, "episode": 10.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a554ab7142f253de1eefcf0997fa0677afc7503234c9af7b5695dcad9295c08c", "size": 1451623, "path": "media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9d248f3bb3a8cadb810ff9d8179f9946a878b0b3d1307fdd347f1c7d79bee5bf", "size": 1468793, "path": "media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "42b418c78bb7f1b89703206b083bfc4242af9452b6a006664e2852861650436b", "size": 1435179, "path": "media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://65an8bxhfflxb22bh5tjp0cz64fq2376jly9yzmbjmtqclonsr4u1ig35ktifwtu48p2dnluyqgxkdj0fjnxbmwv8h7x0svaws5rm6jjk7f7q4sx762vc51x17c8sw29/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://8riqren5j4f3erlet7ri8qqz6bzo6vt4rh7duqtyium92zop6bpzcuboufqu9qmswxeu17pma96wtel4xp84y9zqrogedp8o5t98xffzv8zv8wedlxnka283zjlbnfst:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 10, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.6597032546677664, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.7793639599718221, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.8272221683801428, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.8133141546814853, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "1e52f8fa0b3eaff8361f37df35c643ca7416e4a403526dc0f1ba264fbef828e8", "size": 437, "artifact_path": "wandb-client-artifact://0l04xgg4jit4bwxkbqarza83pe47ab38gz852pqez8ngqx386ubwzgq5y25tmzbukk221zqyjpekdap6fj3nym7pji7ztg5h6g44aqgy55mb8uzm3nqt3augzv54wpy3/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://n48v6rd0bmcxjcn7stho055jvhn4q7z2am0g6t7c5xnm81zqoj0rsgmgxsz28sr0bvx6fikxzjtu7lj0n2rtyatelo8fyd1dqgrdexsf8x8ori4eyz82u1vrtesiw59k:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 284}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.659703254699707, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.7793639302253723, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.8272221684455872, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.8133141398429871, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug-internal.log b/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1473c73fa36877b7f120aa6ad62809174819dcd4 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug-internal.log @@ -0,0 +1,795 @@ +2025-01-03 20:52:36,185 INFO StreamThr :1970913 [internal.py:wandb_internal():85] W&B internal server running at pid: 1970913, started at: 2025-01-03 20:52:36.185175 +2025-01-03 20:52:36,186 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:52:36,198 INFO WriterThread:1970913 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb +2025-01-03 20:52:36,198 DEBUG SenderThread:1970913 [sender.py:send():379] send: header +2025-01-03 20:52:36,207 DEBUG SenderThread:1970913 [sender.py:send():379] send: run +2025-01-03 20:52:36,436 INFO SenderThread:1970913 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files +2025-01-03 20:52:36,436 INFO SenderThread:1970913 [sender.py:_start_run_threads():1188] run started: kdczwpup with start time 1735955556.186292 +2025-01-03 20:52:36,453 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:52:36,454 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:52:36,537 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:52:36,542 DEBUG HandlerThread:1970913 [system_info.py:__init__():26] System info init +2025-01-03 20:52:36,542 DEBUG HandlerThread:1970913 [system_info.py:__init__():41] System info init done +2025-01-03 20:52:36,542 INFO HandlerThread:1970913 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:52:36,542 INFO SystemMonitor:1970913 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:52:36,542 INFO HandlerThread:1970913 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:52:36,543 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:52:36,543 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:52:36,544 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:52:36,544 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:52:36,546 INFO SystemMonitor:1970913 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:52:36,562 DEBUG HandlerThread:1970913 [system_info.py:probe():152] Probing system +2025-01-03 20:52:36,563 DEBUG HandlerThread:1970913 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:probe():200] Probing system done +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:52:36.562203', 'startedAt': '2025-01-04T01:52:36.175252', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.2647500000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4498.229, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 770.2987747192383}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:52:36,568 INFO HandlerThread:1970913 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:52:36,568 INFO HandlerThread:1970913 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:52:36,568 DEBUG HandlerThread:1970913 [system_info.py:_save_code():44] Saving code +2025-01-03 20:52:36,570 DEBUG HandlerThread:1970913 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:52:36,570 DEBUG HandlerThread:1970913 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:52:36,633 DEBUG HandlerThread:1970913 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:52:36,633 INFO HandlerThread:1970913 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:52:36,636 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:52:36,636 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:52:36,636 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:52:36,637 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:52:36,887 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:52:36,887 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:52:36,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:36,888 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/vml0rvyr-wandb-metadata.json +2025-01-03 20:52:36,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:52:36,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:52:36,900 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/k09ugcyl-code/FISH/eval_robot.py +2025-01-03 20:52:36,933 INFO HandlerThread:1970913 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/tb" + save: true +} +control { + relay_id: "ab8a610a85414abebe45349ef347c6c8" +} +_info { + stream_id: "kdczwpup" +} + +2025-01-03 20:52:36,933 DEBUG HandlerThread:1970913 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:52:36,946 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/a2nxh3us-diff.patch +2025-01-03 20:52:37,000 DEBUG SenderThread:1970913 [sender.py:send():379] send: telemetry +2025-01-03 20:52:37,000 DEBUG SenderThread:1970913 [sender.py:send():379] send: tbrecord +2025-01-03 20:52:37,001 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:52:37,001 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 with policy live +2025-01-03 20:52:37,208 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/j1es67v0-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:52:37,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH +2025-01-03 20:52:37,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code +2025-01-03 20:52:37,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:38,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:39,437 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:39,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:40,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:41,518 DEBUG SenderThread:1970913 [sender.py:send():379] send: config +2025-01-03 20:52:41,518 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:41,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:42,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:43,438 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:43,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:44,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:45,439 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:45,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:46,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:46,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:47,440 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:47,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:48,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:49,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:50,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:51,441 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:51,887 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:52:51,888 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:52:51,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:52,003 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:52,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:53,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:54,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:55,442 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:55,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:56,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:57,443 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:57,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:57,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:58,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:59,444 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:52:59,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:00,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:01,444 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:01,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:02,888 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:02,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:03,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:04,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:05,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:06,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:06,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:06,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:07,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:08,015 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:08,446 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:53:08,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:09,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:10,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:11,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:12,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:13,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:13,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:14,895 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:15,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:16,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:17,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:18,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:18,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:19,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:20,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:21,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:21,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:21,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:22,896 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:23,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:24,701 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:24,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:25,451 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:25,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:26,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:27,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:28,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:29,452 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:29,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:29,989 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:30,899 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:31,453 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:31,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:32,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:33,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:34,105 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:53:34,106 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 with policy now +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:53:34,106 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:53:34,107 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:53:34,380 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/zd9xeabn-media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media +2025-01-03 20:53:34,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos +2025-01-03 20:53:34,697 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/2cued4j5-media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:53:34,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:35,108 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:35,454 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:35,900 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:36,546 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:53:36,546 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:53:36,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:36,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:37,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:38,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:39,455 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:39,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:40,408 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:40,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:41,456 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:41,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:42,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:43,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:44,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:45,485 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:45,901 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:46,458 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:53:46,704 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/kq5mzfdj-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:53:46,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:47,458 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:47,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:48,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:49,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:50,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:51,460 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:53:51,466 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:51,889 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:53:51,889 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:53:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:52,902 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:53,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:54,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:55,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:56,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:53:56,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:57,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:58,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:53:59,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:00,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:01,463 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:01,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:01,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:02,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:03,903 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:04,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:05,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:06,554 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:54:06,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:06,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:06,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:06,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:07,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:08,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:09,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:10,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:11,466 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:11,890 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:11,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:12,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:13,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:14,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:15,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:16,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:16,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:17,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:18,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:19,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:20,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:21,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:21,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:21,891 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:21,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:22,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:23,470 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:23,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:24,907 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:25,181 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:54:25,181 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 with policy now +2025-01-03 20:54:25,184 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:54:25,184 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:54:25,184 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:54:25,186 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:54:25,430 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/9zwx3vd7-media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,471 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:54:25,732 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/5f147apn-media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:54:25,908 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:26,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:26,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:27,472 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:27,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:28,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:29,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:30,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:31,473 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:31,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:32,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:32,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:33,473 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:33,909 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:34,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:54:34,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:35,347 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:54:35,347 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:54:35,348 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:54:35,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:54:35,474 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:35,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:36,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:54:36,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:36,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:37,475 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:37,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:37,917 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:38,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:39,476 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:39,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:40,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:41,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:42,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:43,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:43,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:44,911 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:45,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:46,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:47,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:48,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:49,093 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:49,479 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:54:49,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:50,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:51,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:54:51,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:54:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:52,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:53,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:54,912 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:55,049 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:54:55,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:56,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:57,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:58,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:54:59,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:00,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:00,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:01,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:02,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:03,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:04,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:05,894 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:05,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:06,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:55:06,891 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:06,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:06,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:07,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:08,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:09,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:10,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:11,008 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:11,915 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:12,957 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:13,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:13,958 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:55:13,958 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 with policy now +2025-01-03 20:55:14,001 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:55:14,002 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:55:14,002 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:55:14,004 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:55:14,208 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/jx1qz91x-media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,487 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:55:14,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:55:14,813 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/zlwjal09-media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:55:14,921 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:15,488 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:15,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:16,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:16,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:17,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:18,920 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:19,489 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:19,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:20,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:21,490 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:21,893 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:21,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:21,937 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:21,994 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:22,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:23,924 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:24,922 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:25,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:26,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:27,899 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:27,923 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:28,924 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:29,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:30,196 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:55:30,196 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:55:30,197 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:55:30,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:55:30,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:55:30,737 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/votd4cre-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:55:30,925 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:31,493 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:31,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:32,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:33,494 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:33,510 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:33,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:34,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:35,495 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:55:35,926 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:36,548 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:55:36,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:36,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:36,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:37,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:38,927 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:39,003 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:39,928 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:40,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:41,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:42,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:43,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:44,897 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:44,929 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:45,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:46,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:47,930 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:48,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:49,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:49,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:50,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:51,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:55:51,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:55:51,933 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:52,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:53,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:54,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:55,029 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:55:55,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:56,931 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:57,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:58,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:55:59,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:00,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:00,932 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:01,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:02,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:03,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:04,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:05,898 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:05,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:06,549 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:56:06,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:06,892 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:06,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:07,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:08,934 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:09,938 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:10,678 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:56:10,679 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 with policy now +2025-01-03 20:56:10,682 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:56:10,682 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:56:10,682 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:56:10,684 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:56:10,939 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:10,941 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/99skfep5-media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,509 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:56:11,735 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:11,851 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/lkbgp57w-media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:56:11,942 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:12,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:13,510 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:13,943 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:14,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:15,511 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:15,952 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:16,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:16,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:17,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:18,944 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:19,512 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:19,944 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:20,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:21,258 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:56:21,258 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:56:21,258 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:56:21,513 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:56:21,513 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:56:21,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:21,913 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:21,913 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:21,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:22,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:23,514 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:23,947 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:24,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:25,514 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:25,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:26,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:27,515 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:27,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:27,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:28,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:29,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:30,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:31,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:32,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:33,469 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:33,517 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:33,945 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:34,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:35,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:36,550 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:56:36,892 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:36,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:36,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:37,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:38,946 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:39,055 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:39,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:40,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:41,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:42,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:43,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:44,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:44,948 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:45,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:46,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:47,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:48,949 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:49,905 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:49,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:50,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:51,893 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:56:51,893 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:56:51,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:52,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:53,525 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:56:53,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:54,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:55,477 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:56:55,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:56,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:57,950 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:58,951 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:56:59,953 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:00,523 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:00,583 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:00,583 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 with policy now +2025-01-03 20:57:00,586 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:00,586 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:00,586 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:00,593 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:00,855 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/pw2k4bo7-media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:00,956 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:01,528 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval +2025-01-03 20:57:01,823 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/axefb25s-media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:01,960 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:02,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:03,959 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:04,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:05,530 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:05,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:06,385 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:06,550 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:57:06,904 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:57:06,904 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:57:06,963 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:07,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:08,965 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:09,532 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:09,964 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:10,966 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:11,910 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:11,973 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:12,985 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:13,976 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:14,121 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:57:14,121 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table +2025-01-03 20:57:14,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval +2025-01-03 20:57:14,593 INFO wandb-upload_2:1970913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr0euz6my +2025-01-03 20:57:14,775 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/yzyp9d7i-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:14,784 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/w32wdhiq-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:14,966 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:15,001 INFO SenderThread:1970913 [sender.py:send_request_log_artifact():1525] logged artifact run-kdczwpup-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3ODMyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NTE2NA==', 'latestArtifact': None}} +2025-01-03 20:57:15,001 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:15,001 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:57:15,002 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:15,002 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:15,002 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:15,242 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/8f5hsqst-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:15,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:15,534 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:15,968 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:16,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:17,906 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:17,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:18,815 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:18,815 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:18,815 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:18,816 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:18,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:19,080 DEBUG SenderThread:1970913 [sender.py:send():379] send: telemetry +2025-01-03 20:57:19,124 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,124 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,124 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,125 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,125 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,125 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,125 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,126 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,126 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,126 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,126 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,127 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,127 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,127 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,127 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,128 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,128 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,128 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,128 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,129 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,129 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,129 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,184 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:57:19,184 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:57:19,195 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:19,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval +2025-01-03 20:57:19,583 INFO wandb-upload_0:1970913 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp57pjjwrm +2025-01-03 20:57:19,746 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/iyrqw4jc-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:19,928 INFO SenderThread:1970913 [sender.py:send_request_log_artifact():1525] logged artifact run-kdczwpup-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3ODQxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NTE3OQ==', 'latestArtifact': None}} +2025-01-03 20:57:19,928 DEBUG SenderThread:1970913 [sender.py:send():379] send: files +2025-01-03 20:57:19,928 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json with policy now +2025-01-03 20:57:19,929 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:19,929 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:19,929 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:19,967 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:20,157 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/12xjymbx-media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:20,536 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:20,968 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:21,237 DEBUG SenderThread:1970913 [sender.py:send():379] send: exit +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():588] handling runtime: 284 +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_exit():594] send defer +2025-01-03 20:57:21,238 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,238 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:57:21,238 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,238 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:57:21,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:57:21,239 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:57:21,239 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:57:21,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:57:21,239 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:57:21,240 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:57:21,240 DEBUG SystemMonitor:1970913 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:57:21,240 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:57:21,292 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:57:21,293 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:57:21,293 INFO HandlerThread:1970913 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:57:21,293 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,293 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:57:21,293 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:57:21,293 DEBUG SenderThread:1970913 [sender.py:send():379] send: stats +2025-01-03 20:57:21,294 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,294 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:57:21,294 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:21,294 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:57:21,294 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:57:21,294 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:21,294 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:57:21,537 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:21,537 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:23,538 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:27,332 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,333 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:57:27,333 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:27,333 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,333 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:57:27,333 DEBUG SenderThread:1970913 [sender.py:send():379] send: history +2025-01-03 20:57:27,334 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:57:27,335 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:27,335 DEBUG SenderThread:1970913 [sender.py:send():379] send: summary +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:57:27,339 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:57:27,339 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:57:27,339 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,339 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:57:27,340 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,340 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:57:27,343 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:57:27,462 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:57:27,462 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:27,462 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:57:27,462 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:27,462 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:57:27,539 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:27,539 INFO Thread-12 :1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:57:28,239 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:30,753 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:57:30,753 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:30,753 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:30,753 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:57:30,754 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:30,754 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:57:30,754 INFO SenderThread:1970913 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:57:30,756 INFO SenderThread:1970913 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:57:30,758 INFO SenderThread:1970913 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:57:30,770 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:57:30,770 DEBUG SenderThread:1970913 [sender.py:send():379] send: artifact +2025-01-03 20:57:30,770 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:30,773 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 20:57:30,967 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:57:30,967 INFO SenderThread:1970913 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:57:31,240 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:31,540 INFO SenderThread:1970913 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml config.yaml +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json wandb-summary.json +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt requirements.txt +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log output.log +2025-01-03 20:57:31,541 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/diff.patch diff.patch +2025-01-03 20:57:31,542 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 media/videos/eval/2_eval_2_a554ab7142f253de1eef.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 media/videos/eval/4_eval_4_42b418c78bb7f1b89703.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 media/videos/eval/3_eval_3_9d248f3bb3a8cadb810f.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 media/videos/eval/1_eval_1_6117da8fb30ba309d252.mp4 +2025-01-03 20:57:31,547 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 media/videos/eval/0_eval_0_9c48f041424ff0473944.mp4 +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json media/table/eval/total_success_rate_ci_6_1e52f8fa0b3eaff8361f.table.json +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:57:31,550 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:57:31,550 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:31,551 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:31,551 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:57:31,551 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:31,551 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:57:31,551 INFO SenderThread:1970913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:57:31,726 INFO wandb-upload_1:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/requirements.txt +2025-01-03 20:57:31,734 INFO wandb-upload_0:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/wandb-summary.json +2025-01-03 20:57:31,744 INFO wandb-upload_2:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/config.yaml +2025-01-03 20:57:31,778 INFO wandb-upload_4:1970913 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/files/output.log +2025-01-03 20:57:31,843 INFO wandb-upload_3:1970913 [upload_job.py:push():130] Uploaded file /tmp/tmpw2sp7ogywandb/l8txmwyn-events.out.tfevents.1735955556.leonmkim-ROG-Strix-G15CS-G15CS.1970738.0 +2025-01-03 20:57:32,043 INFO Thread-11 (_thread_body):1970913 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:57:32,043 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,043 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:57:32,044 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,044 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:57:32,044 INFO SenderThread:1970913 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:57:32,045 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:57:32,045 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,045 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:57:32,045 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,045 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:57:32,046 INFO SenderThread:1970913 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:57:32,226 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,226 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:57:32,226 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:57:32,226 INFO SenderThread:1970913 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:57:32,226 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:57:32,226 INFO HandlerThread:1970913 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send():379] send: final +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send():379] send: footer +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: defer +2025-01-03 20:57:32,227 INFO SenderThread:1970913 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:57:32,227 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:32,227 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:57:32,228 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:57:32,228 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:57:32,230 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:57:32,230 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:57:32,289 DEBUG SenderThread:1970913 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:57:32,289 INFO MainThread:1970913 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:57:32,290 INFO MainThread:1970913 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:57:32,290 INFO MainThread:1970913 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:57:32,290 DEBUG HandlerThread:1970913 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:57:32,290 INFO HandlerThread:1970913 [handler.py:finish():882] shutting down handler +2025-01-03 20:57:33,228 INFO WriterThread:1970913 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [sender.py:finish():1615] shutting down sender +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:57:33,289 INFO SenderThread:1970913 [file_pusher.py:join():175] waiting for file pusher diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug.log b/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c70f35b2bdaaf2339c7947dd4d87bcbaeb76ee42 --- /dev/null +++ b/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Configure stats pid to 1970738 +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/settings +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug.log +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/wandb/run-20250103_205236-kdczwpup/logs/debug-internal.log +2025-01-03 20:52:36,179 INFO MainThread:1970738 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup'} +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():619] starting backend +2025-01-03 20:52:36,180 INFO MainThread:1970738 [wandb_init.py:init():623] setting up manager +2025-01-03 20:52:36,184 INFO MainThread:1970738 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:52:36,186 INFO MainThread:1970738 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:52:36,198 INFO MainThread:1970738 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:52:36,204 INFO MainThread:1970738 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:52:36,453 INFO MainThread:1970738 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:52:36,491 INFO MainThread:1970738 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:52:36,492 INFO MainThread:1970738 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:52:36,888 INFO MainThread:1970738 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:52:36,890 INFO MainThread:1970738 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:52:36,890 INFO MainThread:1970738 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/kdczwpup/tb, True +2025-01-03 20:52:41,518 INFO MainThread:1970738 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:57:33,291 WARNING MsgRouterThr:1970738 [router.py:message_loop():77] message_loop has been closed diff --git a/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb b/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5acdb166cddcd8d8e1c7ef6f2b0eea91a77cbace Binary files /dev/null and b/kdczwpup/wandb/run-20250103_205236-kdczwpup/run-kdczwpup.wandb differ diff --git a/lmty1tfr/.hydra/config.yaml b/lmty1tfr/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/lmty1tfr/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/lmty1tfr/.hydra/hydra.yaml b/lmty1tfr/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da33d0500225caae10160fb5069b4c0a90b02d58 --- /dev/null +++ b/lmty1tfr/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/lmty1tfr + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/lmty1tfr/.hydra/overrides.yaml b/lmty1tfr/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/lmty1tfr/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/lmty1tfr/eval_policy.log b/lmty1tfr/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..621c3528105921631689f475502540f0ad5592bf --- /dev/null +++ b/lmty1tfr/eval_policy.log @@ -0,0 +1,13 @@ +[2025-01-04 20:25:55,279][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-04 20:25:55,283][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-04 20:25:58,586][rospy.core][INFO] - signal_shutdown [atexit] diff --git a/nbjbj4mc/episode_rosbags/aligned_depth_to_color_K.npy b/nbjbj4mc/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nbjbj4mc/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nbjbj4mc/episode_rosbags/cam_tf_world.npy b/nbjbj4mc/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nbjbj4mc/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nbjbj4mc/episode_rosbags/color_K.npy b/nbjbj4mc/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nbjbj4mc/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nbjbj4mc/episode_rosbags/depth_K.npy b/nbjbj4mc/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nbjbj4mc/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nbjbj4mc/eval_robot.log b/nbjbj4mc/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..66315e614e3c2af4c6584a993e2131ba33736a3e --- /dev/null +++ b/nbjbj4mc/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 21:06:05,067][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 21:06:05,070][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nbjbj4mc/eval_video/1_eval.mp4 b/nbjbj4mc/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..288462e2184760cb1bfdf9649d562aec950b0b91 --- /dev/null +++ b/nbjbj4mc/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9b65874010d74abbc98767ddce7e07e73d19dbc46a2b3e3114003e9206535f +size 903062 diff --git a/nbjbj4mc/eval_video/2_eval.mp4 b/nbjbj4mc/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba39e9e8a43f0d26d3eeb6e1e3127828824722bf --- /dev/null +++ b/nbjbj4mc/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b1ea53d176982b57f123c8b6d3c975c01a76712561b5ba9722fda4b8128dd1 +size 1478324 diff --git a/nbjbj4mc/eval_video/3_eval.mp4 b/nbjbj4mc/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85583933c92b8264db66f25494094f9b3ba8621c --- /dev/null +++ b/nbjbj4mc/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5cf9931a6ec10c216a712dd40e2df87c53c6f7bea3293316f96ff28c1c4176 +size 1481008 diff --git a/nbjbj4mc/eval_video/4_eval.mp4 b/nbjbj4mc/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a22f53ecfdb85d3db2be304302a8afcac6754478 --- /dev/null +++ b/nbjbj4mc/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f0f617b98fb84cf568eea5dda5c50a7bfea1b1b059c26e1f14e15a7ee0cf4b3 +size 1484659 diff --git a/nbjbj4mc/tb/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 b/nbjbj4mc/tb/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 new file mode 100644 index 0000000000000000000000000000000000000000..465ada57ad3fb7641183fd3bbdf4b6f136d8d740 --- /dev/null +++ b/nbjbj4mc/tb/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6eea91fceecc85957a35f75860b0c703c07216355c3b4fa87e2b16e5dddb8a +size 6539 diff --git a/nbjbj4mc/wandb/debug-internal.log b/nbjbj4mc/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..05dd018988347e166b6b72ca3559532d6d18181c --- /dev/null +++ b/nbjbj4mc/wandb/debug-internal.log @@ -0,0 +1,846 @@ +2025-01-03 21:06:10,858 INFO StreamThr :1973165 [internal.py:wandb_internal():85] W&B internal server running at pid: 1973165, started at: 2025-01-03 21:06:10.858332 +2025-01-03 21:06:10,859 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:06:10,871 INFO WriterThread:1973165 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb +2025-01-03 21:06:10,872 DEBUG SenderThread:1973165 [sender.py:send():379] send: header +2025-01-03 21:06:10,880 DEBUG SenderThread:1973165 [sender.py:send():379] send: run +2025-01-03 21:06:11,121 INFO SenderThread:1973165 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files +2025-01-03 21:06:11,121 INFO SenderThread:1973165 [sender.py:_start_run_threads():1188] run started: nbjbj4mc with start time 1735956370.858927 +2025-01-03 21:06:11,136 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:06:11,137 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:06:11,205 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:06:11,210 DEBUG HandlerThread:1973165 [system_info.py:__init__():26] System info init +2025-01-03 21:06:11,210 DEBUG HandlerThread:1973165 [system_info.py:__init__():41] System info init done +2025-01-03 21:06:11,210 INFO HandlerThread:1973165 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:06:11,210 INFO SystemMonitor:1973165 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:06:11,210 INFO HandlerThread:1973165 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:06:11,211 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:06:11,211 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:06:11,212 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:06:11,212 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:06:11,213 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:06:11,223 DEBUG HandlerThread:1973165 [system_info.py:probe():152] Probing system +2025-01-03 21:06:11,225 DEBUG HandlerThread:1973165 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:probe():200] Probing system done +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:06:11.224071', 'startedAt': '2025-01-04T02:06:10.848369', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.3556249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 793.6926231384277}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:06:11,230 INFO HandlerThread:1973165 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:06:11,230 INFO HandlerThread:1973165 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:_save_code():44] Saving code +2025-01-03 21:06:11,231 DEBUG HandlerThread:1973165 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:06:11,231 DEBUG HandlerThread:1973165 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:06:11,293 DEBUG HandlerThread:1973165 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:06:11,294 INFO HandlerThread:1973165 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:06:11,297 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:06:11,297 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:06:11,297 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:06:11,298 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:06:11,530 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/8fwbtakj-wandb-metadata.json +2025-01-03 21:06:11,563 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:06:11,563 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:06:11,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:11,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:11,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:11,566 INFO HandlerThread:1973165 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/tb" + save: true +} +control { + relay_id: "306a0dbd524640ffa6c9b6746b2da61f" +} +_info { + stream_id: "nbjbj4mc" +} + +2025-01-03 21:06:11,567 DEBUG HandlerThread:1973165 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:06:11,574 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/zo5ljkth-diff.patch +2025-01-03 21:06:11,613 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/hllhygdk-code/FISH/eval_robot.py +2025-01-03 21:06:11,652 DEBUG SenderThread:1973165 [sender.py:send():379] send: telemetry +2025-01-03 21:06:11,652 DEBUG SenderThread:1973165 [sender.py:send():379] send: tbrecord +2025-01-03 21:06:11,653 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:06:11,653 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 with policy live +2025-01-03 21:06:11,852 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/7ftj1k1n-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH +2025-01-03 21:06:12,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code +2025-01-03 21:06:12,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:13,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:14,122 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:14,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:15,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:16,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:16,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:17,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:18,259 DEBUG SenderThread:1973165 [sender.py:send():379] send: config +2025-01-03 21:06:18,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:19,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:20,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:20,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:21,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:21,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:22,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:22,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:23,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:24,125 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:24,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:25,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:26,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:26,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:26,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:26,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:27,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:28,126 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:28,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:29,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:30,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:31,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:31,974 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:32,127 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:32,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:33,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:34,128 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:34,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:35,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:36,128 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:36,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:37,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:37,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:38,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:39,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:40,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:41,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:41,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:42,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:42,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:43,130 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:06:43,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:44,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:45,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:46,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:47,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:48,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:48,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:49,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:50,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:51,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:52,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:53,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:53,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:54,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:55,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:56,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:56,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:56,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:57,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:58,134 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:58,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:58,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:59,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:00,134 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:00,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:01,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:02,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:03,577 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:04,055 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:04,136 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:04,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:05,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:06,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:07,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:08,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:09,109 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:09,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:10,043 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:07:10,043 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 with policy now +2025-01-03 21:07:10,043 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:07:10,044 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:07:10,044 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:07:10,044 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:07:10,137 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:07:10,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media +2025-01-03 21:07:10,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos +2025-01-03 21:07:10,330 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/ny4ft7vl-media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:10,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:10,598 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/zpta5879-media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:11,213 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:07:11,214 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:07:11,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:11,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:11,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:12,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:12,576 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:13,576 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:14,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:14,801 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:15,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:16,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:16,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:17,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:18,140 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:18,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:19,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:19,955 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:20,141 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:07:20,386 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/nf84718i-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:07:20,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:21,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:22,141 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:22,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:23,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:24,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:25,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:25,931 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:26,143 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:26,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:26,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:26,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:27,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:28,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:29,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:30,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:31,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:31,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:32,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:33,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:34,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:35,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:36,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:36,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:37,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:38,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:39,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:40,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:41,218 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:07:41,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:41,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:41,726 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:42,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:43,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:44,586 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:44,762 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:07:44,762 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 with policy now +2025-01-03 21:07:44,765 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:07:44,766 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:07:44,766 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:07:44,766 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:07:45,027 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/nnwbe659-media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:07:45,403 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/buom4gvz-media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,589 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:46,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:46,594 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:47,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:47,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:47,594 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:48,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:49,596 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:50,150 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:50,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:51,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:52,151 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:52,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:53,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:53,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:54,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:55,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:56,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:56,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:56,613 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:57,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:58,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:58,684 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:59,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:00,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:01,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:02,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:03,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:04,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:04,598 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:05,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:06,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:07,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:08,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:09,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:10,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:10,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:11,215 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:08:11,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:11,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:11,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:12,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:13,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:14,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:14,988 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:08:14,989 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:08:14,990 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:08:15,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:08:15,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:08:15,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:15,749 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:16,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:16,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:17,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:18,160 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:18,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:19,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:20,160 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:20,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:21,093 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:21,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:22,161 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:22,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:23,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:24,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:25,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:26,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:26,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:26,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:26,662 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:27,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:28,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:29,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:30,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:31,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:32,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:32,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:33,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:34,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:35,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:36,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:37,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:38,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:38,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:39,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:40,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:41,215 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:08:41,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:41,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:42,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:43,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:43,731 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:44,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:45,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:46,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:47,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:48,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:49,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:49,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:50,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:51,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:52,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:53,617 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:54,619 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:54,719 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:55,067 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:08:55,067 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 with policy now +2025-01-03 21:08:55,068 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:08:55,068 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:08:55,069 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:08:55,079 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:08:55,350 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/rv0u9cgd-media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,604 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/5il9et9d-media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:56,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:56,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:56,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:56,624 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:57,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:57,623 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:58,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:59,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:59,877 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:00,173 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:00,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:01,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:02,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:03,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:04,175 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:04,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:05,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:05,678 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:05,844 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:09:05,844 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:09:05,845 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:09:06,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:09:06,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:09:06,417 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/twe7hmyq-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:09:06,629 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:07,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:08,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:08,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:09,669 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:10,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:11,122 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:11,216 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:09:11,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:11,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:11,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:12,178 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:12,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:13,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:14,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:15,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:16,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:16,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:17,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:18,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:19,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:20,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:21,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:22,181 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:22,327 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:22,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:23,629 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:24,182 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:24,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:25,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:26,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:26,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:26,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:27,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:27,710 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:28,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:29,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:30,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:31,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:32,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:33,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:33,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:34,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:35,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:36,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:37,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:38,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:38,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:39,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:40,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:41,216 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:09:41,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:41,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:41,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:42,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:43,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:43,663 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:44,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:45,638 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:45,962 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:09:45,962 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 with policy now +2025-01-03 21:09:45,965 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:09:45,965 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:09:45,966 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:09:45,976 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:09:46,189 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:09:46,231 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/x8conz7x-media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,502 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/ni84j1ja-media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,638 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:47,642 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:48,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:48,645 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:49,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:49,640 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:50,641 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:51,641 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:52,191 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:52,640 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:53,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:54,192 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:54,649 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:55,558 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:55,646 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:56,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:56,575 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:56,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:57,645 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:58,648 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:59,646 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:00,649 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:00,668 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:01,648 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:02,650 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:03,655 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:04,653 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:05,657 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:06,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:06,653 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:07,655 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:08,658 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:09,657 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:10,660 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:11,217 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:10:11,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:11,579 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:11,658 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:11,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:12,662 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:13,660 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:14,667 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:15,671 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:16,668 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:17,587 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:17,670 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:18,675 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:19,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:20,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:21,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:22,588 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:22,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:23,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:24,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:25,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:26,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:26,578 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:26,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:27,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:27,945 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:10:27,945 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:10:27,945 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:27,946 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:28,449 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/g8rmt2db-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:10:28,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:29,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:30,204 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:30,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:31,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:32,204 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:32,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:33,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:33,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:34,205 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:34,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:35,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:36,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:37,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:38,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:38,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:39,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:40,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:41,217 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:10:41,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:41,569 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:41,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:42,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:43,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:43,733 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:44,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:45,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:46,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:47,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:48,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:49,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:49,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:50,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:51,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:52,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:53,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:54,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:54,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:55,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:56,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:56,569 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:56,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:57,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:58,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:59,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:59,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:00,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:01,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:02,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:03,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:04,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:05,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:05,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:06,689 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:07,433 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:07,433 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 with policy now +2025-01-03 21:11:07,444 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:07,445 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:07,445 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:07,446 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:07,684 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:07,714 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/p2vk022b-media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:11:08,499 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/79vq958m-media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,699 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:09,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:09,699 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:10,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:10,704 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:11,213 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:11,218 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:11:11,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:11:11,591 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:11:11,719 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:12,219 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:12,703 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:13,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:14,711 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:15,715 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:16,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:16,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:16,238 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:16,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:16,769 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:11:16,769 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:11:16,907 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:17,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:17,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table +2025-01-03 21:11:17,249 INFO wandb-upload_0:1973165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp77rp_46f +2025-01-03 21:11:17,298 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:11:17,308 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:17,423 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/57072kib-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:17,668 INFO SenderThread:1973165 [sender.py:send_request_log_artifact():1525] logged artifact run-nbjbj4mc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg5NDQ1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NzEzOA==', 'latestArtifact': None}} +2025-01-03 21:11:17,668 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:17,668 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 21:11:17,668 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:17,669 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,669 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,669 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:17,670 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,673 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,673 DEBUG SenderThread:1973165 [sender.py:send():379] send: telemetry +2025-01-03 21:11:17,673 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,674 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,674 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,674 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,674 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,675 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,675 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,676 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,676 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,676 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,676 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,677 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,677 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,677 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,677 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,678 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,678 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:11:17,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:17,910 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/kmmzsua4-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:18,088 INFO wandb-upload_1:1973165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv8pegndb +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval +2025-01-03 21:11:18,562 INFO SenderThread:1973165 [sender.py:send_request_log_artifact():1525] logged artifact run-nbjbj4mc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg5NDQ3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NzE0Ng==', 'latestArtifact': None}} +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:18,562 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json with policy now +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:18,563 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:18,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:18,752 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/c2r0t37b-media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:19,008 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/yuyazn2c-media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:19,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:19,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:20,222 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:20,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:21,563 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:21,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:22,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:23,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:24,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:25,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:26,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:11:26,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:11:26,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:26,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:27,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:28,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:29,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:30,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:31,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:31,855 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:31,897 DEBUG SenderThread:1973165 [sender.py:send():379] send: exit +2025-01-03 21:11:31,897 INFO SenderThread:1973165 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:send_exit():588] handling runtime: 320 +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:send_exit():594] send defer +2025-01-03 21:11:31,898 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,898 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:11:31,898 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:11:31,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:11:31,899 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:11:31,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:11:31,899 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:11:31,899 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:11:31,900 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:11:31,900 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:11:31,952 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:11:31,952 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:11:31,953 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:11:31,953 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,953 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:11:31,953 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:11:31,953 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:11:31,953 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,953 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:11:31,954 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,954 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:11:31,954 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:11:31,954 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,954 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:11:32,226 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:34,227 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:37,229 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:11:38,009 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,009 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:11:38,009 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:38,010 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,011 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:38,011 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:11:38,012 DEBUG SenderThread:1973165 [sender.py:send():379] send: summary +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:38,015 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,016 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:11:38,016 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,016 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:11:38,016 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:11:38,229 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:38,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:40,705 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:11:40,705 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:40,706 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:40,706 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:11:40,706 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:40,706 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:11:40,706 INFO SenderThread:1973165 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:11:40,708 INFO SenderThread:1973165 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:11:40,710 INFO SenderThread:1973165 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:11:40,725 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:11:40,725 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:40,725 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:11:40,725 DEBUG SenderThread:1973165 [sender.py:send():379] send: artifact +2025-01-03 21:11:40,900 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:11:40,943 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml config.yaml +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json wandb-summary.json +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt requirements.txt +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log output.log +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch diff.patch +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:11:41,240 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,240 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,240 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:11:41,240 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,240 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:11:41,240 INFO SenderThread:1973165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:11:41,427 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:41,438 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:11:41,469 INFO wandb-upload_4:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:41,471 INFO wandb-upload_3:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt +2025-01-03 21:11:41,512 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/orgwqikk-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:41,712 INFO Thread-11 (_thread_body):1973165 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:11:41,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,713 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:11:41,713 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,713 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:11:41,713 INFO SenderThread:1973165 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:11:41,714 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:11:41,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,714 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:11:41,714 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,715 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:11:41,715 INFO SenderThread:1973165 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:11:41,900 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:11:41,910 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,910 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,910 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:11:41,910 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:11:41,910 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,910 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send():379] send: final +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send():379] send: footer +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,911 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:11:41,911 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,912 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:11:41,912 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,912 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:11:41,913 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:11:41,914 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:11:41,914 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:41,962 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,962 INFO MainThread:1973165 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:11:41,962 INFO MainThread:1973165 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:11:41,963 INFO MainThread:1973165 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:11:41,963 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:11:41,963 INFO HandlerThread:1973165 [handler.py:finish():882] shutting down handler +2025-01-03 21:11:42,912 INFO WriterThread:1973165 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [sender.py:finish():1615] shutting down sender +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [file_pusher.py:join():175] waiting for file pusher diff --git a/nbjbj4mc/wandb/debug.log b/nbjbj4mc/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f70c836df0d2eac18dbc05f8de8edd404015666e --- /dev/null +++ b/nbjbj4mc/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Configure stats pid to 1972990 +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/settings +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug.log +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug-internal.log +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc'} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():619] starting backend +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():623] setting up manager +2025-01-03 21:06:10,857 INFO MainThread:1972990 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:06:10,858 INFO MainThread:1972990 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:06:10,871 INFO MainThread:1972990 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:06:10,878 INFO MainThread:1972990 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:06:11,136 INFO MainThread:1972990 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:06:11,200 INFO MainThread:1972990 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:06:11,200 INFO MainThread:1972990 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:06:11,564 INFO MainThread:1972990 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:06:11,566 INFO MainThread:1972990 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:06:11,566 INFO MainThread:1972990 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/tb, True +2025-01-03 21:06:18,259 INFO MainThread:1972990 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:11:42,964 WARNING MsgRouterThr:1972990 [router.py:message_loop():77] message_loop has been closed diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8266f1e76302287be1c506a7b786ff93336374b9 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735956370 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..260fed1aee926a75f7c67b83fb8ec5b323d21693 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..703ec3d 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..410bbb6812c99af7f4b7968c39fdbb4d8bf11e3d --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[17, 20, 0.85, 0.5937137603572338, 0.9727514832266396, 0.6565820127412438, 0.9672621673901157, 0.6905067562675931, 0.9430732055612381, 0.7050621360130267, 0.9409921447795794]]} \ No newline at end of file diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1a04e1f69bba6af770e4f81fb586c70dd35de28b --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f77bab9262745cc63038c9e69bef37e3a42647fab820df96630d9433416dccc8 +size 1455301 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..288462e2184760cb1bfdf9649d562aec950b0b91 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9b65874010d74abbc98767ddce7e07e73d19dbc46a2b3e3114003e9206535f +size 903062 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ba39e9e8a43f0d26d3eeb6e1e3127828824722bf --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b1ea53d176982b57f123c8b6d3c975c01a76712561b5ba9722fda4b8128dd1 +size 1478324 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85583933c92b8264db66f25494094f9b3ba8621c --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5cf9931a6ec10c216a712dd40e2df87c53c6f7bea3293316f96ff28c1c4176 +size 1481008 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a22f53ecfdb85d3db2be304302a8afcac6754478 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f0f617b98fb84cf568eea5dda5c50a7bfea1b1b059c26e1f14e15a7ee0cf4b3 +size 1484659 diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c930d08b1634b4deb24c7a4bfe23660e0dc7f147 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log @@ -0,0 +1,291 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735956378.260007]: resetting environment +[INFO] [1735956378.264132]: cleared current plan +[INFO] [1735956378.265086]: moving to home +[INFO] [1735956379.267713]: reached home +[INFO] [1735956379.268885]: reset action history +[INFO] [1735956381.979846]: environment reset +Starting episode 0 +[INFO] [1735956381.984064]: resetting environment +[INFO] [1735956381.991106]: cleared current plan +[INFO] [1735956381.992296]: moving to home +[INFO] [1735956382.994538]: reached home +[INFO] [1735956382.995767]: reset action history +[INFO] [1735956385.703373]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735956390.963410]: resetting environment +[INFO] [1735956390.967639]: cleared current plan +[INFO] [1735956390.970751]: moving to home +[INFO] [1735956391.973459]: reached home +[INFO] [1735956391.975184]: reset action history +[INFO] [1735956394.683272]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/episode_rosbags/episode_0_2025-01-03-21-06-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1972990.log +started roslaunch server http://158.130.50.37:34507/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1973271] +process[print_text-2]: started with pid [1973295] +process[rosbag_record-3]: started with pid [1973296] +started bagging! +For topic gripper_width: timestamp difference is -106270457 for nearest: 1735956415861588096 - target: 1735956415967858553 at idx 249 +For topic gripper_width: timestamp difference is 128320925 for nearest: 1735956417628251068 - target: 1735956417499930143 at idx 249 +For topic EE_pose: timestamp difference is 146127092 for nearest: 1735956422575509655 - target: 1735956422429382563 at idx 0 +For topic EE_pose: timestamp difference is 177142999 for nearest: 1735956422606525562 - target: 1735956422429382563 at idx 0 +For topic EE_pose: timestamp difference is 241103217 for nearest: 1735956422670485780 - target: 1735956422429382563 at idx 0 +[INFO] [1735956429.109021]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1735956429.836189]: Stored episode 1. +[INFO] [1735956429.836461]: Saving video... +[INFO] [1735956430.027946]: Video saved! +Starting episode 1 +[INFO] [1735956430.044828]: resetting environment +[INFO] [1735956430.054320]: cleared current plan +[INFO] [1735956430.054772]: moving to home +[INFO] [1735956433.757252]: reached home +[INFO] [1735956433.757708]: reset action history +[INFO] [1735956436.469013]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956439.959141]: resetting environment +[INFO] [1735956439.964911]: cleared current plan +[INFO] [1735956439.965134]: moving to home +[INFO] [1735956440.965599]: reached home +[INFO] [1735956440.966087]: reset action history +[INFO] [1735956443.674877]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/episode_rosbags/episode_1_2025-01-03-21-07-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1972990.log +started roslaunch server http://158.130.50.37:36619/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1973421] +process[print_text-5]: started with pid [1973422] +process[rosbag_record-6]: started with pid [1973423] +started bagging! +preempting episode +[INFO] [1735956463.349136]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735956464.129641]: Stored episode 2. +[INFO] [1735956464.129855]: Saving video... +[INFO] [1735956464.730513]: Video saved! +Starting episode 2 +[INFO] [1735956464.772497]: resetting environment +[INFO] [1735956464.812137]: cleared current plan +[INFO] [1735956464.815365]: moving to home +[WARN] [1735956464.478405]: Plan exhausted +[WARN] [1735956464.518228]: Plan exhausted +[WARN] [1735956464.558815]: Plan exhausted +[WARN] [1735956464.598343]: Plan exhausted +[WARN] [1735956464.638262]: Plan exhausted +[WARN] [1735956464.678675]: Plan exhausted +[WARN] [1735956464.718491]: Plan exhausted +[WARN] [1735956464.759569]: Plan exhausted +[WARN] [1735956464.798207]: Plan exhausted +[INFO] [1735956468.427735]: reached home +[INFO] [1735956468.429976]: reset action history +[INFO] [1735956471.222189]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956494.731724]: resetting environment +[INFO] [1735956494.747814]: cleared current plan +[INFO] [1735956494.748132]: moving to home +[INFO] [1735956495.749289]: reached home +[INFO] [1735956495.749733]: reset action history +[INFO] [1735956498.463190]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/episode_rosbags/episode_2_2025-01-03-21-08-18.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1972990.log +started roslaunch server http://158.130.50.37:40749/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1973572] +process[print_text-8]: started with pid [1973573] +process[rosbag_record-9]: started with pid [1973597] +started bagging! +For topic gripper_width: timestamp difference is -105172083 for nearest: 1735956499861582353 - target: 1735956499966754436 at idx 23 +[INFO] [1735956533.381314]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1735956534.353469]: Stored episode 3. +[INFO] [1735956534.360558]: Saving video... +[INFO] [1735956535.033237]: Video saved! +Starting episode 3 +[INFO] [1735956535.075274]: resetting environment +[INFO] [1735956535.108675]: cleared current plan +[INFO] [1735956535.123848]: moving to home +[WARN] [1735956534.719326]: Plan exhausted +[WARN] [1735956534.759337]: Plan exhausted +[WARN] [1735956534.800221]: Plan exhausted +[WARN] [1735956534.842241]: Plan exhausted +[WARN] [1735956534.882740]: Plan exhausted +[WARN] [1735956534.919717]: Plan exhausted +[WARN] [1735956534.961242]: Plan exhausted +[WARN] [1735956534.999965]: Plan exhausted +[WARN] [1735956535.042232]: Plan exhausted +[WARN] [1735956535.079315]: Plan exhausted +[INFO] [1735956538.833564]: reached home +[INFO] [1735956538.836205]: reset action history +[INFO] [1735956541.631905]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956545.867802]: resetting environment +[INFO] [1735956545.888641]: cleared current plan +[INFO] [1735956545.889011]: moving to home +[INFO] [1735956546.892339]: reached home +[INFO] [1735956546.895267]: reset action history +[INFO] [1735956549.625574]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/episode_rosbags/episode_3_2025-01-03-21-09-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1972990.log +started roslaunch server http://158.130.50.37:33215/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1973736] +process[print_text-11]: started with pid [1973737] +process[rosbag_record-12]: started with pid [1973738] +started bagging! +For topic gripper_width: timestamp difference is 110912982 for nearest: 1735956561028247539 - target: 1735956560917334557 at idx 160 +For topic gripper_width: timestamp difference is 113536636 for nearest: 1735956562328247587 - target: 1735956562214710951 at idx 178 +[INFO] [1735956584.703183]: Storing episode... +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[INFO] [1735956585.428678]: Stored episode 4. +[INFO] [1735956585.434251]: Saving video... +[INFO] [1735956585.909731]: Video saved! +Starting episode 4 +[INFO] [1735956585.970714]: resetting environment +[INFO] [1735956586.004085]: cleared current plan +[INFO] [1735956586.005167]: moving to home +[INFO] [1735956589.709529]: reached home +[INFO] [1735956589.716531]: reset action history +[INFO] [1735956592.513065]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735956627.371591]: resetting environment +[INFO] [1735956627.393752]: cleared current plan +[INFO] [1735956627.394935]: moving to home +[INFO] [1735956628.400070]: reached home +[INFO] [1735956628.403817]: reset action history +[INFO] [1735956631.186429]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/episode_rosbags/episode_4_2025-01-03-21-10-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1972990.log +started roslaunch server http://158.130.50.37:36011/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1973856] +process[print_text-14]: started with pid [1973857] +process[rosbag_record-15]: started with pid [1973881] +started bagging! +[INFO] [1735956666.079892]: Storing episode... +[rosbag_record-15] killing on exit[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735956666.878633]: Stored episode 5. +[INFO] [1735956666.890946]: Saving video... +[INFO] [1735956667.377360]: Video saved! +[INFO] [1735956667.454462]: resetting environment +[WARN] [1735956667.402255]: Plan exhausted +[WARN] [1735956667.440084]: Plan exhausted +[WARN] [1735956667.486249]: Plan exhausted +[INFO] [1735956667.495197]: cleared current plan +[INFO] [1735956667.495748]: moving to home +[INFO] [1735956671.206203]: reached home +[INFO] [1735956671.218666]: reset action history +[INFO] [1735956674.026246]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b1b3e436e87b64b74c02f25c7b81e00f1e229b1f --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T02:06:11.224071", + "startedAt": "2025-01-04T02:06:10.848369", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.3556249999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.978, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 793.6926231384277 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..064634982aa18c9cab0e36151893c4918a71116e --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "f77bab9262745cc63038c9e69bef37e3a42647fab820df96630d9433416dccc8", "size": 1455301, "path": "media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4"}, "global_step": 606, "_timestamp": 1735956677.1957557, "_runtime": 306.3368287086487, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "6f9b65874010d74abbc98767ddce7e07e73d19dbc46a2b3e3114003e9206535f", "size": 903062, "path": "media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4"}, "eval/num_success": 4.0, "episode": 20.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "89b1ea53d176982b57f123c8b6d3c975c01a76712561b5ba9722fda4b8128dd1", "size": 1478324, "path": "media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "7f5cf9931a6ec10c216a712dd40e2df87c53c6f7bea3293316f96ff28c1c4176", "size": 1481008, "path": "media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "9f0f617b98fb84cf568eea5dda5c50a7bfea1b1b059c26e1f14e15a7ee0cf4b3", "size": 1484659, "path": "media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://wylcx9yg72xo0lmv9u437gojsazjj9p1sgjs4yh1rih3r6m0m43gpdyit0v36kg31p15d6qmn1fddbiug5xxwldxd7nhxahv0y57x65q1ww1358enfjod21x22ql13gs/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ljp8x49nxrpta8bm8nbzulao5h65pvhj4qal2438ldvscaanqwa6lpcmtl7y15xwxkbazazxf1335tsxgdd5c7hj6y5jd087ecqm7ayhcb7v8g6apu3488wigl9wxkfa:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 17, "total_num_episodes": 20, "total_success_rate": 0.85, "total_success_rate_lb_0.01": 0.5937137603572338, "total_success_rate_ub_0.01": 0.9727514832266396, "total_success_rate_lb_0.025": 0.6565820127412438, "total_success_rate_ub_0.025": 0.9672621673901157, "total_success_rate_lb_0.05": 0.6905067562675931, "total_success_rate_ub_0.05": 0.9430732055612381, "total_success_rate_lb_0.1": 0.7050621360130267, "total_success_rate_ub_0.1": 0.9409921447795794, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "4a909e78c5c4561dc11b7239d33c0beedcf1604921a0e261a12be7d8fda9ed8b", "size": 506, "artifact_path": "wandb-client-artifact://exwzfelc6b8ovpyslsc7kbblwsxztxchq1ht1hi8tqpv7wz7j8hfomlyp15ododxr2ku3b761afwl5y09juyqj27mmsbl4elsk40ba7o4pze2m7uqnxkmmk6fcaxuuk2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://04f22cfqmxfk2zgufys26v9wmhddxt1uhyexifsc5a6fvlk10ewdcavaf5vxzuce2r45ld73as08qbg5m494d7zds90rrfrxi0o7s88x8uf6ecmoohiu6jac3bslccnm:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 320}, "eval/total_success_rate": 0.8500000238418579, "eval/total_success_rate_lb_0.01": 0.5937137603759766, "eval/total_success_rate_ub_0.01": 0.9727514982223511, "eval/total_success_rate_lb_0.025": 0.6565819978713989, "eval/total_success_rate_ub_0.025": 0.9672621488571167, "eval/total_success_rate_lb_0.05": 0.6905067563056946, "eval/total_success_rate_ub_0.05": 0.9430732131004333, "eval/total_success_rate_lb_0.1": 0.7050621509552002, "eval/total_success_rate_ub_0.1": 0.9409921169281006} \ No newline at end of file diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug-internal.log b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..05dd018988347e166b6b72ca3559532d6d18181c --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug-internal.log @@ -0,0 +1,846 @@ +2025-01-03 21:06:10,858 INFO StreamThr :1973165 [internal.py:wandb_internal():85] W&B internal server running at pid: 1973165, started at: 2025-01-03 21:06:10.858332 +2025-01-03 21:06:10,859 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status +2025-01-03 21:06:10,871 INFO WriterThread:1973165 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb +2025-01-03 21:06:10,872 DEBUG SenderThread:1973165 [sender.py:send():379] send: header +2025-01-03 21:06:10,880 DEBUG SenderThread:1973165 [sender.py:send():379] send: run +2025-01-03 21:06:11,121 INFO SenderThread:1973165 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files +2025-01-03 21:06:11,121 INFO SenderThread:1973165 [sender.py:_start_run_threads():1188] run started: nbjbj4mc with start time 1735956370.858927 +2025-01-03 21:06:11,136 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 21:06:11,137 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: check_version +2025-01-03 21:06:11,205 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 21:06:11,210 DEBUG HandlerThread:1973165 [system_info.py:__init__():26] System info init +2025-01-03 21:06:11,210 DEBUG HandlerThread:1973165 [system_info.py:__init__():41] System info init done +2025-01-03 21:06:11,210 INFO HandlerThread:1973165 [system_monitor.py:start():194] Starting system monitor +2025-01-03 21:06:11,210 INFO SystemMonitor:1973165 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 21:06:11,210 INFO HandlerThread:1973165 [system_monitor.py:probe():214] Collecting system info +2025-01-03 21:06:11,211 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 21:06:11,211 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started disk monitoring +2025-01-03 21:06:11,212 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 21:06:11,212 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started memory monitoring +2025-01-03 21:06:11,213 INFO SystemMonitor:1973165 [interfaces.py:start():188] Started network monitoring +2025-01-03 21:06:11,223 DEBUG HandlerThread:1973165 [system_info.py:probe():152] Probing system +2025-01-03 21:06:11,225 DEBUG HandlerThread:1973165 [system_info.py:_probe_git():137] Probing git +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:_probe_git():145] Probing git done +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:probe():200] Probing system done +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T02:06:11.224071', 'startedAt': '2025-01-04T02:06:10.848369', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.3556249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.978, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 793.6926231384277}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 21:06:11,230 INFO HandlerThread:1973165 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 21:06:11,230 INFO HandlerThread:1973165 [system_monitor.py:probe():227] Publishing system info +2025-01-03 21:06:11,230 DEBUG HandlerThread:1973165 [system_info.py:_save_code():44] Saving code +2025-01-03 21:06:11,231 DEBUG HandlerThread:1973165 [system_info.py:_save_code():67] Saving code done +2025-01-03 21:06:11,231 DEBUG HandlerThread:1973165 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 21:06:11,293 DEBUG HandlerThread:1973165 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 21:06:11,294 INFO HandlerThread:1973165 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 21:06:11,297 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:06:11,297 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 21:06:11,297 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 21:06:11,298 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 21:06:11,530 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/8fwbtakj-wandb-metadata.json +2025-01-03 21:06:11,563 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 21:06:11,563 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: python_packages +2025-01-03 21:06:11,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:11,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:11,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:11,566 INFO HandlerThread:1973165 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/tb" + save: true +} +control { + relay_id: "306a0dbd524640ffa6c9b6746b2da61f" +} +_info { + stream_id: "nbjbj4mc" +} + +2025-01-03 21:06:11,567 DEBUG HandlerThread:1973165 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 21:06:11,574 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/zo5ljkth-diff.patch +2025-01-03 21:06:11,613 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/hllhygdk-code/FISH/eval_robot.py +2025-01-03 21:06:11,652 DEBUG SenderThread:1973165 [sender.py:send():379] send: telemetry +2025-01-03 21:06:11,652 DEBUG SenderThread:1973165 [sender.py:send():379] send: tbrecord +2025-01-03 21:06:11,653 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:06:11,653 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 with policy live +2025-01-03 21:06:11,852 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/7ftj1k1n-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json +2025-01-03 21:06:12,123 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH +2025-01-03 21:06:12,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code +2025-01-03 21:06:12,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:13,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:14,122 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:14,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:15,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:16,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:16,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:17,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:18,259 DEBUG SenderThread:1973165 [sender.py:send():379] send: config +2025-01-03 21:06:18,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:19,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:20,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:20,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:21,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:21,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:22,124 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:22,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:23,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:24,125 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:24,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:25,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:26,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:26,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:26,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:26,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:27,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:28,126 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:28,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:29,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:30,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:31,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:31,974 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:32,127 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:32,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:33,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:34,128 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:34,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:35,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:36,128 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:36,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:37,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:37,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:38,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:39,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:40,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:41,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:41,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:42,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:42,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:43,130 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:06:43,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:44,570 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:45,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:46,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:47,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:48,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:48,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:49,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:50,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:51,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:52,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:53,566 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:53,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:54,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:55,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:56,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:06:56,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:06:56,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:57,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:58,134 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:06:58,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:06:58,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:06:59,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:00,134 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:00,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:01,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:02,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:03,577 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:04,055 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:04,136 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:04,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:05,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:06,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:07,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:08,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:09,109 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:09,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:10,043 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:07:10,043 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 with policy now +2025-01-03 21:07:10,043 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:07:10,044 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:07:10,044 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:07:10,044 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:07:10,137 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:10,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:07:10,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media +2025-01-03 21:07:10,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos +2025-01-03 21:07:10,330 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/ny4ft7vl-media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:10,575 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:10,598 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/zpta5879-media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:07:11,213 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 21:07:11,214 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:07:11,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:11,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:11,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:12,138 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:12,576 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:13,576 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:14,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:14,801 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:15,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:16,139 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:16,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:17,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:18,140 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:18,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:19,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:19,955 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:20,141 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:07:20,386 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/nf84718i-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:07:20,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:21,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:22,141 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:22,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:23,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:24,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:25,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:25,931 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:26,143 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:26,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:26,564 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:26,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:27,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:28,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:29,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:30,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:31,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:31,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:32,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:33,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:34,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:35,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:36,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:36,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:37,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:38,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:39,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:40,581 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:41,218 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:07:41,564 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:41,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:41,726 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:42,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:43,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:44,586 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:44,762 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:07:44,762 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 with policy now +2025-01-03 21:07:44,765 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:07:44,766 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:07:44,766 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:07:44,766 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:07:45,027 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/nnwbe659-media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:07:45,403 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/buom4gvz-media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:07:45,589 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:46,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:46,594 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:47,149 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:47,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:47,594 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:48,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:49,596 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:50,150 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:50,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:51,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:52,151 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:07:52,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:53,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:53,590 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:54,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:55,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:56,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:07:56,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:07:56,613 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:57,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:58,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:07:58,684 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:07:59,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:00,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:01,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:02,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:03,592 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:04,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:04,598 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:05,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:06,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:07,597 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:08,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:09,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:10,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:10,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:11,215 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:08:11,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:11,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:11,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:12,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:13,599 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:14,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:14,988 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:08:14,989 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:08:14,990 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:08:15,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:08:15,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:08:15,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:15,749 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:16,159 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:16,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:17,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:18,160 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:18,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:19,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:20,160 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:20,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:21,093 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:21,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:22,161 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:22,600 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:23,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:24,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:25,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:26,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:26,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:26,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:26,662 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:27,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:28,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:29,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:30,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:31,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:32,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:32,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:33,601 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:34,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:35,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:36,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:37,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:38,573 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:38,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:39,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:40,602 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:41,215 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:08:41,565 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:41,565 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:41,609 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:42,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:43,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:43,731 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:44,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:45,603 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:46,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:47,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:48,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:49,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:49,604 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:50,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:51,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:52,605 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:53,617 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:54,619 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:54,719 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:08:55,067 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:08:55,067 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 with policy now +2025-01-03 21:08:55,068 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:08:55,068 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:08:55,069 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:08:55,079 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:08:55,350 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/rv0u9cgd-media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,604 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/5il9et9d-media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:08:55,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:56,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:56,571 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:08:56,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:08:56,624 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:57,172 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:08:57,623 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:58,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:59,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:08:59,877 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:00,173 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:00,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:01,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:02,625 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:03,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:04,175 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:04,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:05,626 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:05,678 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:05,844 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:09:05,844 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:09:05,845 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:09:06,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:09:06,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:09:06,417 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/twe7hmyq-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:09:06,629 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:07,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:08,176 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:08,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:09,669 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:10,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:11,122 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:11,216 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:09:11,567 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:11,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:11,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:12,178 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:12,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:13,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:14,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:15,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:16,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:16,627 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:17,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:18,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:19,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:20,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:21,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:22,181 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:22,327 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:22,628 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:23,629 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:24,182 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:24,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:25,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:26,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:26,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:26,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:27,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:27,710 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:28,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:29,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:30,630 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:31,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:32,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:33,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:33,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:34,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:35,631 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:36,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:37,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:38,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:38,632 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:39,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:40,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:41,216 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:09:41,568 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:41,568 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:41,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:42,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:43,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:43,663 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:44,633 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:45,638 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:45,962 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:09:45,962 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 with policy now +2025-01-03 21:09:45,965 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:09:45,965 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:09:45,966 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:09:45,976 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:09:46,189 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:09:46,231 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/x8conz7x-media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,502 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/ni84j1ja-media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:09:46,638 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:47,642 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:48,190 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:48,645 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:49,580 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:49,640 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:50,641 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:51,641 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:52,191 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:52,640 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:53,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:54,192 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:09:54,649 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:55,558 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:09:55,646 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:56,574 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:09:56,575 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:09:56,644 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:57,645 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:58,648 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:09:59,646 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:00,649 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:00,668 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:01,648 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:02,650 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:03,655 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:04,653 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:05,657 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:06,582 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:06,653 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:07,655 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:08,658 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:09,657 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:10,660 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:11,217 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:10:11,579 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:11,579 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:11,658 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:11,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:12,662 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:13,660 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:14,667 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:15,671 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:16,668 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:17,587 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:17,670 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:18,675 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:19,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:20,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:21,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:22,588 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:22,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:23,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:24,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:25,677 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:26,578 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:26,578 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:26,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:27,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:27,945 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:10:27,945 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:10:27,945 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:27,946 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:10:28,203 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:28,449 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/g8rmt2db-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:10:28,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:29,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:30,204 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:30,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:31,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:32,204 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:32,679 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:33,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:33,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:34,205 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:10:34,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:35,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:36,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:37,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:38,583 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:38,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:39,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:40,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:41,217 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:10:41,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:41,569 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:41,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:42,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:43,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:43,733 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:44,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:45,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:46,680 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:47,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:48,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:49,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:49,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:50,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:51,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:52,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:53,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:54,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:54,681 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:55,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:56,569 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:10:56,569 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:10:56,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:57,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:58,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:10:59,673 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:10:59,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:00,682 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:01,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:02,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:03,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:04,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:05,584 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:05,683 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:06,689 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:07,433 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:07,433 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 with policy now +2025-01-03 21:11:07,444 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:07,445 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:07,445 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:07,446 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:07,684 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:07,714 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/p2vk022b-media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval +2025-01-03 21:11:08,499 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/79vq958m-media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:08,699 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:09,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:09,699 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:10,217 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:10,704 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:11,213 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:11,218 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:11:11,591 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:11:11,591 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:11:11,719 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:12,219 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:12,703 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:13,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:14,711 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:15,715 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:16,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:16,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:16,238 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:16,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:16,769 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:11:16,769 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:11:16,907 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:17,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:17,220 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval +2025-01-03 21:11:17,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table +2025-01-03 21:11:17,249 INFO wandb-upload_0:1973165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp77rp_46f +2025-01-03 21:11:17,298 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 21:11:17,308 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 21:11:17,423 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/57072kib-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:17,668 INFO SenderThread:1973165 [sender.py:send_request_log_artifact():1525] logged artifact run-nbjbj4mc-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg5NDQ1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NzEzOA==', 'latestArtifact': None}} +2025-01-03 21:11:17,668 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:17,668 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 21:11:17,668 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:17,669 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,669 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,669 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:17,670 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,673 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,673 DEBUG SenderThread:1973165 [sender.py:send():379] send: telemetry +2025-01-03 21:11:17,673 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,674 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,674 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,674 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,674 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,675 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,675 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,675 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,676 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,676 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,676 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,676 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,677 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,677 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,677 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,677 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,678 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:17,678 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:17,678 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 21:11:17,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:17,910 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/kmmzsua4-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:18,088 INFO wandb-upload_1:1973165 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpv8pegndb +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:18,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval +2025-01-03 21:11:18,562 INFO SenderThread:1973165 [sender.py:send_request_log_artifact():1525] logged artifact run-nbjbj4mc-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg5NDQ3Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NzE0Ng==', 'latestArtifact': None}} +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send():379] send: files +2025-01-03 21:11:18,562 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json with policy now +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:18,562 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:18,563 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:18,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:18,752 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/c2r0t37b-media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:19,008 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/yuyazn2c-media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:19,221 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:19,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:20,222 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:20,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:21,563 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:21,712 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:22,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:23,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:24,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:25,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:26,572 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 21:11:26,572 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: stop_status +2025-01-03 21:11:26,672 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:26,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:27,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:28,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:29,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:30,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:31,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:31,855 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:31,897 DEBUG SenderThread:1973165 [sender.py:send():379] send: exit +2025-01-03 21:11:31,897 INFO SenderThread:1973165 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:send_exit():588] handling runtime: 320 +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:31,898 INFO SenderThread:1973165 [sender.py:send_exit():594] send defer +2025-01-03 21:11:31,898 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,898 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 21:11:31,898 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 1 +2025-01-03 21:11:31,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 21:11:31,899 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 21:11:31,899 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 2 +2025-01-03 21:11:31,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 21:11:31,899 INFO HandlerThread:1973165 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 21:11:31,899 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 21:11:31,899 DEBUG SystemMonitor:1973165 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 21:11:31,900 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 21:11:31,900 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 21:11:31,952 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 21:11:31,952 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 21:11:31,953 INFO HandlerThread:1973165 [interfaces.py:finish():200] Joined network monitor +2025-01-03 21:11:31,953 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,953 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 21:11:31,953 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 3 +2025-01-03 21:11:31,953 DEBUG SenderThread:1973165 [sender.py:send():379] send: stats +2025-01-03 21:11:31,953 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,953 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 21:11:31,954 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:31,954 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 21:11:31,954 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 4 +2025-01-03 21:11:31,954 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:31,954 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 21:11:32,226 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:34,227 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:37,229 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:38,008 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 21:11:38,009 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,009 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 21:11:38,009 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 5 +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send():379] send: history +2025-01-03 21:11:38,010 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: summary_record +2025-01-03 21:11:38,010 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,011 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:38,011 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 21:11:38,012 DEBUG SenderThread:1973165 [sender.py:send():379] send: summary +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 21:11:38,015 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 21:11:38,015 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 6 +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,016 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 21:11:38,016 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 7 +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 21:11:38,016 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:38,016 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 21:11:38,016 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:38,016 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 21:11:38,229 INFO Thread-12 :1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:38,899 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:40,705 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 8 +2025-01-03 21:11:40,705 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:40,706 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:40,706 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 21:11:40,706 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:40,706 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 21:11:40,706 INFO SenderThread:1973165 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 21:11:40,708 INFO SenderThread:1973165 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 21:11:40,710 INFO SenderThread:1973165 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 21:11:40,725 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 9 +2025-01-03 21:11:40,725 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:40,725 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 21:11:40,725 DEBUG SenderThread:1973165 [sender.py:send():379] send: artifact +2025-01-03 21:11:40,900 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'versionIndex': 408}}} +2025-01-03 21:11:40,943 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 21:11:40,943 INFO SenderThread:1973165 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files +2025-01-03 21:11:41,230 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml config.yaml +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json wandb-summary.json +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt requirements.txt +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-metadata.json wandb-metadata.json +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log output.log +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/diff.patch diff.patch +2025-01-03 21:11:41,231 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 media/videos/eval/0_eval_0_f77bab9262745cc63038.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 media/videos/eval/3_eval_3_7f5cf9931a6ec10c216a.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 media/videos/eval/4_eval_4_9f0f617b98fb84cf568e.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 media/videos/eval/2_eval_2_89b1ea53d176982b57f1.mp4 +2025-01-03 21:11:41,235 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 media/videos/eval/1_eval_1_6f9b65874010d74abbc9.mp4 +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json media/table/eval/total_success_rate_ci_6_4a909e78c5c4561dc11b.table.json +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 21:11:41,239 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 10 +2025-01-03 21:11:41,240 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,240 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,240 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 21:11:41,240 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,240 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 21:11:41,240 INFO SenderThread:1973165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:11:41,427 INFO wandb-upload_2:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/wandb-summary.json +2025-01-03 21:11:41,438 INFO wandb-upload_0:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/config.yaml +2025-01-03 21:11:41,469 INFO wandb-upload_4:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/output.log +2025-01-03 21:11:41,471 INFO wandb-upload_3:1973165 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/files/requirements.txt +2025-01-03 21:11:41,512 INFO wandb-upload_1:1973165 [upload_job.py:push():130] Uploaded file /tmp/tmpse2_cyp6wandb/orgwqikk-events.out.tfevents.1735956371.leonmkim-ROG-Strix-G15CS-G15CS.1972990.0 +2025-01-03 21:11:41,712 INFO Thread-11 (_thread_body):1973165 [sender.py:transition_state():614] send defer: 11 +2025-01-03 21:11:41,713 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,713 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 21:11:41,713 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,713 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 21:11:41,713 INFO SenderThread:1973165 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 21:11:41,714 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 12 +2025-01-03 21:11:41,714 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,714 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 21:11:41,714 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,715 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 21:11:41,715 INFO SenderThread:1973165 [file_stream.py:finish():601] file stream finish called +2025-01-03 21:11:41,900 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [file_stream.py:finish():605] file stream finish is done +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 13 +2025-01-03 21:11:41,910 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,910 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,910 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 21:11:41,910 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 21:11:41,910 INFO SenderThread:1973165 [sender.py:transition_state():614] send defer: 14 +2025-01-03 21:11:41,910 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: defer +2025-01-03 21:11:41,910 INFO HandlerThread:1973165 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send():379] send: final +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send():379] send: footer +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: defer +2025-01-03 21:11:41,911 INFO SenderThread:1973165 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 21:11:41,911 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,911 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,912 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 21:11:41,912 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 21:11:41,912 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: server_info +2025-01-03 21:11:41,913 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 21:11:41,914 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 21:11:41,914 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 21:11:41,962 DEBUG SenderThread:1973165 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 21:11:41,962 INFO MainThread:1973165 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 21:11:41,962 INFO MainThread:1973165 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 21:11:41,963 INFO MainThread:1973165 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 21:11:41,963 DEBUG HandlerThread:1973165 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 21:11:41,963 INFO HandlerThread:1973165 [handler.py:finish():882] shutting down handler +2025-01-03 21:11:42,912 INFO WriterThread:1973165 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [sender.py:finish():1615] shutting down sender +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 21:11:42,962 INFO SenderThread:1973165 [file_pusher.py:join():175] waiting for file pusher diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug.log b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f70c836df0d2eac18dbc05f8de8edd404015666e --- /dev/null +++ b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Configure stats pid to 1972990 +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/settings +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug.log +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/logs/debug-internal.log +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():569] calling init triggers +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc'} +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():619] starting backend +2025-01-03 21:06:10,850 INFO MainThread:1972990 [wandb_init.py:init():623] setting up manager +2025-01-03 21:06:10,857 INFO MainThread:1972990 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 21:06:10,858 INFO MainThread:1972990 [wandb_init.py:init():631] backend started and connected +2025-01-03 21:06:10,871 INFO MainThread:1972990 [wandb_init.py:init():720] updated telemetry +2025-01-03 21:06:10,878 INFO MainThread:1972990 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 21:06:11,136 INFO MainThread:1972990 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 21:06:11,200 INFO MainThread:1972990 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 21:06:11,200 INFO MainThread:1972990 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 21:06:11,564 INFO MainThread:1972990 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 21:06:11,565 INFO MainThread:1972990 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 21:06:11,566 INFO MainThread:1972990 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 21:06:11,566 INFO MainThread:1972990 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nbjbj4mc/tb, True +2025-01-03 21:06:18,259 INFO MainThread:1972990 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-03 21:11:42,964 WARNING MsgRouterThr:1972990 [router.py:message_loop():77] message_loop has been closed diff --git a/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..838f423a8522becd06e1cdaa21b3e7db084f4715 Binary files /dev/null and b/nbjbj4mc/wandb/run-20250103_210610-nbjbj4mc/run-nbjbj4mc.wandb differ diff --git a/nibxxtai/.hydra/config.yaml b/nibxxtai/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/nibxxtai/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/nibxxtai/.hydra/hydra.yaml b/nibxxtai/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2844f59fdb43a66f8d6b31473d17b0d59296d258 --- /dev/null +++ b/nibxxtai/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/nibxxtai/.hydra/overrides.yaml b/nibxxtai/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/nibxxtai/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/nibxxtai/episode_rosbags/aligned_depth_to_color_K.npy b/nibxxtai/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nibxxtai/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nibxxtai/episode_rosbags/cam_tf_world.npy b/nibxxtai/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/nibxxtai/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/nibxxtai/episode_rosbags/color_K.npy b/nibxxtai/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/nibxxtai/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/nibxxtai/episode_rosbags/depth_K.npy b/nibxxtai/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/nibxxtai/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/nibxxtai/eval_robot.log b/nibxxtai/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..16b4ef4d9583f41f762d2613ab48bdac4978f6f1 --- /dev/null +++ b/nibxxtai/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 20:46:15,045][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 20:46:15,048][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/nibxxtai/eval_video/0_eval.mp4 b/nibxxtai/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..222ea8543fcbf89e68fedb33828b9bb063c1dc12 --- /dev/null +++ b/nibxxtai/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24fb1057b6de7362aaa79fcef8740ccacc5f656588e8af34b02af5f23c6cb53d +size 1462076 diff --git a/nibxxtai/eval_video/1_eval.mp4 b/nibxxtai/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1be449c5fd1ebb9cd5b1b7a048d652222c706f38 --- /dev/null +++ b/nibxxtai/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4e7b679afe12f99a3d283f7e1460e3766c59287e6cf7afc0880391f0304178 +size 1430279 diff --git a/nibxxtai/eval_video/2_eval.mp4 b/nibxxtai/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5823d4762304e4f697445174585d69c33853bb60 --- /dev/null +++ b/nibxxtai/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:121e578fe04e49712301b992b8c88fbbf37a7dde04b742e67142910d9f54d907 +size 1437569 diff --git a/nibxxtai/eval_video/3_eval.mp4 b/nibxxtai/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd7d539f02f60fc7ac5ce226dcfcc8387f862616 --- /dev/null +++ b/nibxxtai/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3410e3f59909246de3a196da0af2947609031539b7b7d6d452f9cfba91e9ac1 +size 1474124 diff --git a/nibxxtai/eval_video/4_eval.mp4 b/nibxxtai/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e72e23ded4e63d8ce7b9991603c7f88fc990c84d --- /dev/null +++ b/nibxxtai/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5d9702636bb8a8301f723ad1eb64eab410f9b1317a8f87e4ea78c156a009dd +size 1438049 diff --git a/nibxxtai/tb/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 b/nibxxtai/tb/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 new file mode 100644 index 0000000000000000000000000000000000000000..7ec6b5d095bd53166ef361dcefdb7cce6d93cd46 --- /dev/null +++ b/nibxxtai/tb/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbc3d7c2c1b72924c113ad7c0c5f88aa532c41c8d6c863f9c321ecf045f942a +size 6539 diff --git a/nibxxtai/wandb/debug-internal.log b/nibxxtai/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3e0072668f945bee5c1292d780717d13e96aefad --- /dev/null +++ b/nibxxtai/wandb/debug-internal.log @@ -0,0 +1,882 @@ +2025-01-03 20:46:21,085 INFO StreamThr :1969795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1969795, started at: 2025-01-03 20:46:21.084702 +2025-01-03 20:46:21,087 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:46:21,099 INFO WriterThread:1969795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb +2025-01-03 20:46:21,099 DEBUG SenderThread:1969795 [sender.py:send():379] send: header +2025-01-03 20:46:21,106 DEBUG SenderThread:1969795 [sender.py:send():379] send: run +2025-01-03 20:46:21,327 INFO SenderThread:1969795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files +2025-01-03 20:46:21,327 INFO SenderThread:1969795 [sender.py:_start_run_threads():1188] run started: nibxxtai with start time 1735955181.085192 +2025-01-03 20:46:21,341 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:46:21,341 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:46:21,420 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:46:21,424 DEBUG HandlerThread:1969795 [system_info.py:__init__():26] System info init +2025-01-03 20:46:21,424 DEBUG HandlerThread:1969795 [system_info.py:__init__():41] System info init done +2025-01-03 20:46:21,425 INFO HandlerThread:1969795 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:46:21,425 INFO SystemMonitor:1969795 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:46:21,425 INFO HandlerThread:1969795 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:46:21,426 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:46:21,426 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:46:21,427 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:46:21,427 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:46:21,428 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:46:21,438 DEBUG HandlerThread:1969795 [system_info.py:probe():152] Probing system +2025-01-03 20:46:21,439 DEBUG HandlerThread:1969795 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:probe():200] Probing system done +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:46:21.438190', 'startedAt': '2025-01-04T01:46:21.074705', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.484375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 758.1241111755371}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:46:21,444 INFO HandlerThread:1969795 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:46:21,444 INFO HandlerThread:1969795 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:_save_code():44] Saving code +2025-01-03 20:46:21,446 DEBUG HandlerThread:1969795 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:46:21,446 DEBUG HandlerThread:1969795 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:46:21,508 DEBUG HandlerThread:1969795 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:46:21,509 INFO HandlerThread:1969795 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:46:21,512 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:46:21,747 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/qjr3og54-wandb-metadata.json +2025-01-03 20:46:21,755 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:46:21,755 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:46:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:21,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:21,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:21,759 INFO HandlerThread:1969795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/tb" + save: true +} +control { + relay_id: "f961af653af74e62a8f20ce7de4ace91" +} +_info { + stream_id: "nibxxtai" +} + +2025-01-03 20:46:21,759 DEBUG HandlerThread:1969795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:46:21,801 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/4yn8ctry-code/FISH/eval_robot.py +2025-01-03 20:46:21,814 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/a6a562hm-diff.patch +2025-01-03 20:46:21,863 DEBUG SenderThread:1969795 [sender.py:send():379] send: telemetry +2025-01-03 20:46:21,863 DEBUG SenderThread:1969795 [sender.py:send():379] send: tbrecord +2025-01-03 20:46:21,864 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:46:21,864 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 with policy live +2025-01-03 20:46:22,041 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ilo1kr6v-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH +2025-01-03 20:46:22,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:23,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:24,328 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:24,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:25,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:26,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:26,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:27,347 DEBUG SenderThread:1969795 [sender.py:send():379] send: config +2025-01-03 20:46:27,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:28,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:29,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:30,330 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:30,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:31,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:31,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:32,330 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:32,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:33,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:34,331 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:34,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:35,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:36,331 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:36,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:36,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:36,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:36,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:37,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:38,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:39,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:40,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:41,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:42,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:42,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:43,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:44,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:45,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:46,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:47,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:47,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:48,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:49,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:50,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:51,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:51,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:51,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:52,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:52,853 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:53,336 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:46:53,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:54,337 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:54,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:55,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:56,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:57,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:58,338 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:58,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:58,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:59,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:00,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:01,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:02,339 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:02,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:03,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:04,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:04,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:05,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:06,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:06,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:06,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:07,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:08,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:09,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:09,914 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:10,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:11,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:12,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:13,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:14,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:15,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:15,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:16,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:17,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:18,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:19,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:20,343 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:20,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:21,428 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:47:21,429 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:47:21,429 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:21,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:21,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:22,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:23,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:24,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:25,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:26,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:26,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:27,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:28,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:29,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:30,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:31,743 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:47:31,743 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 with policy now +2025-01-03 20:47:31,743 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:47:31,744 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:47:31,744 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:47:31,744 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:47:31,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:31,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:31,969 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/aj97t357-media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,346 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos +2025-01-03 20:47:32,628 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ojk5b6dv-media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:33,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:34,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:34,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:35,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:36,348 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:36,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:36,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:36,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:36,894 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:37,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:38,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:39,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:40,349 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:40,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:41,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:42,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:42,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:43,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:44,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:45,351 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:47:45,579 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/fjkpnicu-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:47:45,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:46,351 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:46,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:47,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:48,352 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:48,457 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:48,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:49,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:50,352 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:50,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:51,429 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:47:51,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:51,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:51,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:52,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:53,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:53,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:54,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:55,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:56,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:57,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:58,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:59,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:59,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:00,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:01,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:02,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:03,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:04,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:04,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:05,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:06,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:06,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:06,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:07,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:08,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:09,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:09,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:10,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:11,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:12,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:13,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:14,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:15,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:15,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:16,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:17,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:18,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:19,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:20,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:20,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:21,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:48:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:21,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:21,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:22,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:23,775 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:24,357 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:48:24,357 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 with policy now +2025-01-03 20:48:24,357 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:48:24,358 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:48:24,358 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:48:24,360 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:48:24,627 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/4zwkjt5i-media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:24,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:24,885 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/wduc7cuc-media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:25,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:26,364 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:26,378 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:26,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:27,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:28,776 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:29,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:30,365 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:30,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:31,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:31,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:32,366 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:32,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:33,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:34,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:35,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:36,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:36,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:36,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:36,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:37,367 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:48:37,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:38,191 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:48:38,191 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:48:38,191 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:48:38,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:48:38,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:38,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:39,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:40,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:40,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:41,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:42,369 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:42,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:42,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:43,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:44,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:45,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:46,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:47,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:47,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:48,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:49,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:50,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:51,430 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:48:51,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:51,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:51,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:52,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:52,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:53,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:54,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:55,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:56,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:57,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:58,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:58,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:59,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:00,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:01,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:02,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:03,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:04,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:04,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:05,790 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:06,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:06,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:06,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:07,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:08,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:09,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:09,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:10,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:11,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:12,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:13,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:14,379 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:14,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:15,509 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:15,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:16,380 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:16,798 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:17,277 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:49:17,277 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 with policy now +2025-01-03 20:49:17,278 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:49:17,278 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:49:17,279 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:49:17,282 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:49:17,380 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:49:17,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:49:17,492 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/7f8w5him-media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,748 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/kyniqsxz-media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,800 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:18,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:18,802 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:19,804 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:20,807 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:21,118 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:21,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:49:21,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:21,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:21,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:22,382 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:22,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:23,808 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:24,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:25,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:26,384 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:26,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:26,811 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:27,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:28,817 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:29,815 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:30,818 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:31,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:31,823 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:32,826 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:33,819 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:34,822 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:35,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:36,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:36,765 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:36,823 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:36,919 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:37,824 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:38,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:39,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:40,824 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:41,825 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:42,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:42,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:43,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:44,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:45,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:46,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:47,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:48,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:48,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:49,837 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:50,833 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:51,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:49:51,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:51,765 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:51,834 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:52,838 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:53,837 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:53,910 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:54,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:55,840 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:56,841 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:57,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:58,841 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:59,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:59,842 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:00,845 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:01,073 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:50:01,073 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:50:01,074 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:50:01,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:50:01,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:50:01,635 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/gdtcxzwk-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:50:01,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:02,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:02,844 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:03,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:04,397 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:04,845 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:05,689 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:05,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:06,398 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:06,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:06,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:06,846 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:07,846 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:08,399 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:08,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:09,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:10,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:10,914 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:11,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:12,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:13,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:14,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:15,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:16,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:16,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:17,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:18,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:19,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:20,403 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:20,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:21,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:50:21,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:21,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:21,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:21,899 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:22,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:23,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:24,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:25,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:26,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:27,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:27,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:28,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:29,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:30,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:31,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:32,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:32,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:33,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:34,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:35,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:36,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:36,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:36,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:37,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:37,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:38,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:39,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:40,409 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:40,851 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:41,753 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:50:41,753 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 with policy now +2025-01-03 20:50:41,755 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:50:41,755 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:50:41,755 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:50:41,758 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:50:41,851 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:50:42,601 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/uby4ka6c-media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:42,859 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:42,865 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/8kzzppl9-media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:43,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:43,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:44,411 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:44,862 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:45,866 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:46,411 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:46,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:47,862 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:48,866 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:49,377 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:49,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:50,413 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:50,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:51,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:50:51,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:51,762 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:51,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:52,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:53,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:54,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:54,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:55,870 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:56,871 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:57,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:58,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:59,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:00,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:00,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:01,144 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:01,144 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:01,146 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:01,416 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:01,416 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:01,657 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/zw5r0y49-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:01,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:02,417 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:02,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:03,882 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:04,417 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:04,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:05,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:06,110 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:06,418 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:06,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:06,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:06,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:07,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:08,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:09,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:10,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:11,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:11,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:12,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:13,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:14,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:15,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:16,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:16,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:17,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:18,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:19,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:20,424 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:20,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:21,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:51:21,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:21,761 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:21,832 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:21,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:22,425 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:22,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:23,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:24,426 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:24,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:25,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:26,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:27,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:27,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:28,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:29,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:30,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:31,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:32,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:33,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:33,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:34,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:35,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:36,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:36,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:36,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:37,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:38,820 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:38,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:39,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:40,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:40,878 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:41,217 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:41,217 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 with policy now +2025-01-03 20:51:41,220 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:41,221 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:41,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:41,229 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:51:41,437 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/mhv620ni-media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,693 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/2c0sloev-media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,883 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:42,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:42,888 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:43,433 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:43,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:44,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:44,890 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:45,899 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:46,434 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:46,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:47,905 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:48,901 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:49,894 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:49,905 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:50,435 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:50,893 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:50,997 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:51:50,997 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:51:51,207 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval +2025-01-03 20:51:51,470 INFO wandb-upload_0:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8oaaost9 +2025-01-03 20:51:51,564 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:51:51,572 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:51,613 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/647pa3dy-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:51,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:51,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:51,907 INFO SenderThread:1969795 [sender.py:send_request_log_artifact():1525] logged artifact run-nibxxtai-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjAyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NDM0Mg==', 'latestArtifact': None}} +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:51,907 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,908 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,908 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:51,908 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,912 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send():379] send: telemetry +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,913 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,914 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,914 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,914 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,914 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,915 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,915 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,915 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,915 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,916 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,916 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,917 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,917 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,917 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,917 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,918 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,918 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,918 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,918 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:51:52,143 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/wi43gvgz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:52,285 INFO wandb-upload_2:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1pcwp_2j +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:52,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:52,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval +2025-01-03 20:51:52,829 INFO SenderThread:1969795 [sender.py:send_request_log_artifact():1525] logged artifact run-nibxxtai-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjA0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NDM0OQ==', 'latestArtifact': None}} +2025-01-03 20:51:52,829 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:52,830 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-03 20:51:52,830 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:52,830 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:52,831 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:52,831 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:52,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:53,049 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/kkjskfeh-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:53,281 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/5exuhlr2-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:53,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:53,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:54,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:54,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:54,963 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:55,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:56,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:57,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:58,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:59,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:00,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:00,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:01,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:02,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:03,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:04,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:05,625 DEBUG SenderThread:1969795 [sender.py:send():379] send: exit +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():588] handling runtime: 344 +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():594] send defer +2025-01-03 20:52:05,626 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,626 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:52:05,627 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:52:05,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:52:05,627 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:52:05,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:52:05,627 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:52:05,628 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:52:05,628 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:52:05,628 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:52:05,682 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:52:05,683 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,683 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:52:05,683 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:52:05,683 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:52:05,683 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:52:05,684 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,684 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:52:05,684 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:52:05,684 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,684 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:52:06,442 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:06,442 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:12,213 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:12,213 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,214 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:52:12,214 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,214 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,215 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,215 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:52:12,215 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:52:12,216 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:52:12,216 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:12,217 DEBUG SenderThread:1969795 [sender.py:send():379] send: summary +2025-01-03 20:52:12,220 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:12,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:52:12,221 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,221 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:52:12,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:52:12,225 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,348 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:52:12,348 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,348 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:52:12,348 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,348 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:52:12,444 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:12,444 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:52:12,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:15,064 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:52:15,065 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:15,065 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:15,065 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:52:15,065 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:15,065 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:52:15,065 INFO SenderThread:1969795 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:52:15,068 INFO SenderThread:1969795 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:52:15,070 INFO SenderThread:1969795 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:52:15,084 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:52:15,084 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:15,084 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:52:15,084 DEBUG SenderThread:1969795 [sender.py:send():379] send: artifact +2025-01-03 20:52:15,446 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:15,502 INFO wandb-upload_2:1969795 [upload_job.py:push():85] Skipped uploading /tmp/tmpb1t3b0z7/wandb-job.json +2025-01-03 20:52:15,502 INFO wandb-upload_1:1969795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaz_huuo5 +2025-01-03 20:52:15,597 INFO wandb-upload_0:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgtghzagk +2025-01-03 20:52:15,628 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'versionIndex': 407}}} +2025-01-03 20:52:16,172 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:52:16,446 INFO SenderThread:1969795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml config.yaml +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json wandb-summary.json +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:52:16,448 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt requirements.txt +2025-01-03 20:52:16,448 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:52:16,449 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log output.log +2025-01-03 20:52:16,449 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch diff.patch +2025-01-03 20:52:16,450 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:52:16,451 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:52:16,452 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:52:16,455 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:52:16,456 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,456 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,456 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:52:16,456 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,457 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:52:16,457 INFO SenderThread:1969795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:52:16,618 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:16,628 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,629 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,676 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ekl1azpc-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:52:16,678 INFO wandb-upload_4:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:16,678 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:52:16,679 INFO wandb-upload_3:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt +2025-01-03 20:52:16,879 INFO Thread-11 (_thread_body):1969795 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:52:16,880 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,880 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:52:16,880 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,880 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:52:16,880 INFO SenderThread:1969795 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:52:16,881 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:52:16,881 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,881 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:52:16,882 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,882 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:52:16,882 INFO SenderThread:1969795 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:52:16,948 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,948 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:52:16,948 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:52:16,948 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,948 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send():379] send: final +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send():379] send: footer +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,949 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:52:16,949 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:52:16,950 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:52:16,951 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:17,000 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:17,000 INFO MainThread:1969795 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:52:17,001 INFO MainThread:1969795 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:52:17,001 INFO MainThread:1969795 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:52:17,001 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:52:17,001 INFO HandlerThread:1969795 [handler.py:finish():882] shutting down handler +2025-01-03 20:52:17,950 INFO WriterThread:1969795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb +2025-01-03 20:52:18,000 INFO SenderThread:1969795 [sender.py:finish():1615] shutting down sender +2025-01-03 20:52:18,001 INFO SenderThread:1969795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:52:18,001 INFO SenderThread:1969795 [file_pusher.py:join():175] waiting for file pusher diff --git a/nibxxtai/wandb/debug.log b/nibxxtai/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4a05699ecce95a7c72de19df4e59b5b5b2915054 --- /dev/null +++ b/nibxxtai/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Configure stats pid to 1969621 +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/settings +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug.log +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug-internal.log +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai'} +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():619] starting backend +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():623] setting up manager +2025-01-03 20:46:21,083 INFO MainThread:1969621 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:46:21,085 INFO MainThread:1969621 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:46:21,098 INFO MainThread:1969621 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:46:21,104 INFO MainThread:1969621 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:46:21,341 INFO MainThread:1969621 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:46:21,414 INFO MainThread:1969621 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:46:21,414 INFO MainThread:1969621 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:46:21,756 INFO MainThread:1969621 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:46:21,758 INFO MainThread:1969621 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:46:21,758 INFO MainThread:1969621 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/tb, True +2025-01-03 20:46:27,347 INFO MainThread:1969621 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:52:18,002 WARNING MsgRouterThr:1969621 [router.py:message_loop():77] message_loop has been closed diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a74a0736a311685937445c906a42f0aafcfdc753 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735955181 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..260fed1aee926a75f7c67b83fb8ec5b323d21693 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..703ec3d 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,19 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_2' # seed 2 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd62798ae2246ea155c51d2c7f88d715ecd78c3 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 5, 1.0, 0.43520843983992363, 1, 0.60740816591022, 1, 0.6842965185273651, 1, 0.6614799052153926, 1]]} \ No newline at end of file diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..222ea8543fcbf89e68fedb33828b9bb063c1dc12 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24fb1057b6de7362aaa79fcef8740ccacc5f656588e8af34b02af5f23c6cb53d +size 1462076 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1be449c5fd1ebb9cd5b1b7a048d652222c706f38 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4e7b679afe12f99a3d283f7e1460e3766c59287e6cf7afc0880391f0304178 +size 1430279 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5823d4762304e4f697445174585d69c33853bb60 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:121e578fe04e49712301b992b8c88fbbf37a7dde04b742e67142910d9f54d907 +size 1437569 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fd7d539f02f60fc7ac5ce226dcfcc8387f862616 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3410e3f59909246de3a196da0af2947609031539b7b7d6d452f9cfba91e9ac1 +size 1474124 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e72e23ded4e63d8ce7b9991603c7f88fc990c84d --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d5d9702636bb8a8301f723ad1eb64eab410f9b1317a8f87e4ea78c156a009dd +size 1438049 diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dce2fed7b544998d9b220977fb01e46c62efc526 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log @@ -0,0 +1,279 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735955187.347703]: resetting environment +[INFO] [1735955187.355130]: cleared current plan +[INFO] [1735955187.356011]: moving to home +[INFO] [1735955188.358475]: reached home +[INFO] [1735955188.360296]: reset action history +[INFO] [1735955191.070645]: environment reset +Starting episode 0 +[INFO] [1735955191.077676]: resetting environment +[INFO] [1735955191.086607]: cleared current plan +[INFO] [1735955191.087588]: moving to home +[INFO] [1735955192.089219]: reached home +[INFO] [1735955192.090470]: reset action history +[INFO] [1735955194.799760]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735955211.842945]: resetting environment +[INFO] [1735955211.847202]: cleared current plan +[INFO] [1735955211.847979]: moving to home +[INFO] [1735955212.850305]: reached home +[INFO] [1735955212.852188]: reset action history +[INFO] [1735955215.559494]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/episode_rosbags/episode_0_2025-01-03-20-46-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1969621.log +started roslaunch server http://158.130.50.37:40471/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1969901] +process[print_text-2]: started with pid [1969925] +process[rosbag_record-3]: started with pid [1969926] +started bagging! +For topic gripper_width: timestamp difference is -105481364 for nearest: 1735955220361553691 - target: 1735955220467035055 at idx 69 +For topic gripper_width: timestamp difference is 111892686 for nearest: 1735955237528261886 - target: 1735955237416369200 at idx 249 +[INFO] [1735955250.783886]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735955251.534569]: Stored episode 1. +[INFO] [1735955251.535106]: Saving video... +[INFO] [1735955251.729368]: Video saved! +Starting episode 1 +[INFO] [1735955251.745519]: resetting environment +[INFO] [1735955251.751558]: cleared current plan +[INFO] [1735955251.752010]: moving to home +[INFO] [1735955255.354832]: reached home +[INFO] [1735955255.356045]: reset action history +[INFO] [1735955258.066481]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955264.742035]: resetting environment +[INFO] [1735955264.746973]: cleared current plan +[INFO] [1735955264.747710]: moving to home +[INFO] [1735955265.749452]: reached home +[INFO] [1735955265.749868]: reset action history +[INFO] [1735955268.456504]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/episode_rosbags/episode_1_2025-01-03-20-47-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1969621.log +started roslaunch server http://158.130.50.37:33085/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1970049] +process[print_text-5]: started with pid [1970050] +process[rosbag_record-6]: started with pid [1970051] +started bagging! +For topic gripper_width: timestamp difference is 100554337 for nearest: 1735955269294948687 - target: 1735955269194394350 at idx 15 +[INFO] [1735955303.285056]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735955304.012227]: Stored episode 2. +[INFO] [1735955304.014265]: Saving video... +[INFO] [1735955304.329738]: Video saved! +Starting episode 2 +[INFO] [1735955304.359613]: resetting environment +[INFO] [1735955304.377275]: cleared current plan +[INFO] [1735955304.377657]: moving to home +[INFO] [1735955308.181527]: reached home +[INFO] [1735955308.185079]: reset action history +[INFO] [1735955310.972157]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955317.187998]: resetting environment +[INFO] [1735955317.204370]: cleared current plan +[INFO] [1735955317.207540]: moving to home +[INFO] [1735955318.209274]: reached home +[INFO] [1735955318.210407]: reset action history +[INFO] [1735955320.919230]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/episode_rosbags/episode_2_2025-01-03-20-48-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1969621.log +started roslaunch server http://158.130.50.37:46531/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1970192] +process[print_text-8]: started with pid [1970193] +process[rosbag_record-9]: started with pid [1970217] +started bagging! +For topic gripper_width: timestamp difference is 102933676 for nearest: 1735955351828264744 - target: 1735955351725331068 at idx 249 +For topic gripper_width: timestamp difference is 102644855 for nearest: 1735955354461592370 - target: 1735955354358947515 at idx 249 +[INFO] [1735955356.025050]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit[print_text-8] killing on exit +[INFO] [1735955356.812973]: Stored episode 3. +[INFO] [1735955356.814345]: Saving video... +[INFO] [1735955357.239443]: Video saved! +Starting episode 3 +[INFO] [1735955357.281222]: resetting environment +[INFO] [1735955357.308327]: cleared current plan +[INFO] [1735955357.311197]: moving to home +[INFO] [1735955361.117503]: reached home +[INFO] [1735955361.120072]: reset action history +[INFO] [1735955363.897286]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955400.910553]: resetting environment +[INFO] [1735955400.919401]: cleared current plan +[INFO] [1735955400.919606]: moving to home +[INFO] [1735955401.921333]: reached home +[INFO] [1735955401.923803]: reset action history +[INFO] [1735955404.646886]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/episode_rosbags/episode_3_2025-01-03-20-50-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1969621.log +started roslaunch server http://158.130.50.37:40959/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1970407] +process[print_text-11]: started with pid [1970408] +process[rosbag_record-12]: started with pid [1970409] +started bagging! +For topic gripper_width: timestamp difference is 150478609 for nearest: 1735955418828260668 - target: 1735955418677782059 at idx 183 +For topic gripper_width: timestamp difference is 104682309 for nearest: 1735955439428279740 - target: 1735955439323597431 at idx 249 +[INFO] [1735955440.700823]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735955441.430680]: Stored episode 4. +[INFO] [1735955441.433231]: Saving video... +[INFO] [1735955441.726180]: Video saved! +Starting episode 4 +[INFO] [1735955441.760066]: resetting environment +[INFO] [1735955441.810522]: cleared current plan +[INFO] [1735955441.811769]: moving to home +[INFO] [1735955445.518363]: reached home +[INFO] [1735955445.530537]: reset action history +[INFO] [1735955448.333132]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735955460.955190]: resetting environment +[INFO] [1735955460.987579]: cleared current plan +[INFO] [1735955460.993243]: moving to home +[INFO] [1735955461.997205]: reached home +[INFO] [1735955462.002245]: reset action history +[INFO] [1735955464.788183]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/episode_rosbags/episode_4_2025-01-03-20-51-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1969621.log +started roslaunch server http://158.130.50.37:43045/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1970526] +process[print_text-14]: started with pid [1970551] +process[rosbag_record-15]: started with pid [1970552] +started bagging! +For topic gripper_width: timestamp difference is 101120223 for nearest: 1735955478928261508 - target: 1735955478827141285 at idx 172 +For topic gripper_width: timestamp difference is 103000238 for nearest: 1735955479462393596 - target: 1735955479359393358 at idx 176 +For topic gripper_width: timestamp difference is 106401206 for nearest: 1735955481528258801 - target: 1735955481421857595 at idx 203 +For topic gripper_width: timestamp difference is 108991929 for nearest: 1735955482828213521 - target: 1735955482719221592 at idx 221 +[INFO] [1735955499.687281]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735955500.536176]: Stored episode 5. +[INFO] [1735955500.544715]: Saving video... +[INFO] [1735955501.156228]: Video saved! +[INFO] [1735955501.221848]: resetting environment +[INFO] [1735955501.253636]: cleared current plan +[INFO] [1735955501.256272]: moving to home +[WARN] [1735955501.030332]: Plan exhausted +[WARN] [1735955501.069731]: Plan exhausted +[WARN] [1735955501.113136]: Plan exhausted +[WARN] [1735955501.152427]: Plan exhausted +[WARN] [1735955501.189483]: Plan exhausted +[WARN] [1735955501.229147]: Plan exhausted +[INFO] [1735955505.068344]: reached home +[INFO] [1735955505.078461]: reset action history +[INFO] [1735955507.863129]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..62b81303f7ba6cb356e0afbf9621ef2559b09381 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T01:46:21.438190", + "startedAt": "2025-01-04T01:46:21.074705", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.484375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.991, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 758.1241111755371 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab5047b03d530ffbb68c6943a2bdf01d02e2c --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "24fb1057b6de7362aaa79fcef8740ccacc5f656588e8af34b02af5f23c6cb53d", "size": 1462076, "path": "media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4"}, "global_step": 665, "_timestamp": 1735955511.4645627, "_runtime": 330.3793706893921, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "da4e7b679afe12f99a3d283f7e1460e3766c59287e6cf7afc0880391f0304178", "size": 1430279, "path": "media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "121e578fe04e49712301b992b8c88fbbf37a7dde04b742e67142910d9f54d907", "size": 1437569, "path": "media/videos/eval/2_eval_2_121e578fe04e49712301.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a3410e3f59909246de3a196da0af2947609031539b7b7d6d452f9cfba91e9ac1", "size": 1474124, "path": "media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "1d5d9702636bb8a8301f723ad1eb64eab410f9b1317a8f87e4ea78c156a009dd", "size": 1438049, "path": "media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://ugp2qbv5dlk2oywb8vnggh0kuvluzo40xrb29jf7wyo3ahvne7hdg7tzcc5sjg30vt80t3kc27kasez9svrqq1fp7f08ceyh5wrfbbc6xasnymq7lsy7zdy1he1yf56j/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://zggqasuidyofpid1ozvub5ps577x3r6rf26imq5ir7b0nycakg9qb830wpn7bhpkx8p8v2r6bqp15xwt8sjs247yaosdx49c4hq2ymja4uvvhjvj6aa85zud0a1vb6lv:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 5, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.43520843983992363, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.60740816591022, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.6842965185273651, "total_success_rate_ub_0.05": 1, "total_success_rate_lb_0.1": 0.6614799052153926, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "e32f206babc2825ce21715b938b57354f64231e089822aca554fad29505cf476", "size": 434, "artifact_path": "wandb-client-artifact://w518y8dzn37bi0peanttgcdldxitevabrbq1wiq27vfgj782ric6h6dpojjx8oo7l9u05teoear31jbrl5yggdubrfy1qx7un586tv7j5ivl9qwjj5mz1si6ykrmp1gv/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://2t2y4xw1xdxur01r6310ckb2girzrmmlm1pkmjq1evba4vryrdlbnqcco6ygbw7r4zd4javufz9gaux4e4oiaj95bishtyvx227anoam71n9ef175xokx6n4s4i2d5bs:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 344}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.43520843982696533, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.6074081659317017, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.6842964887619019, "eval/total_success_rate_ub_0.05": 1.0, "eval/total_success_rate_lb_0.1": 0.6614798903465271, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug-internal.log b/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3e0072668f945bee5c1292d780717d13e96aefad --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug-internal.log @@ -0,0 +1,882 @@ +2025-01-03 20:46:21,085 INFO StreamThr :1969795 [internal.py:wandb_internal():85] W&B internal server running at pid: 1969795, started at: 2025-01-03 20:46:21.084702 +2025-01-03 20:46:21,087 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status +2025-01-03 20:46:21,099 INFO WriterThread:1969795 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb +2025-01-03 20:46:21,099 DEBUG SenderThread:1969795 [sender.py:send():379] send: header +2025-01-03 20:46:21,106 DEBUG SenderThread:1969795 [sender.py:send():379] send: run +2025-01-03 20:46:21,327 INFO SenderThread:1969795 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files +2025-01-03 20:46:21,327 INFO SenderThread:1969795 [sender.py:_start_run_threads():1188] run started: nibxxtai with start time 1735955181.085192 +2025-01-03 20:46:21,341 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 20:46:21,341 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: check_version +2025-01-03 20:46:21,420 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 20:46:21,424 DEBUG HandlerThread:1969795 [system_info.py:__init__():26] System info init +2025-01-03 20:46:21,424 DEBUG HandlerThread:1969795 [system_info.py:__init__():41] System info init done +2025-01-03 20:46:21,425 INFO HandlerThread:1969795 [system_monitor.py:start():194] Starting system monitor +2025-01-03 20:46:21,425 INFO SystemMonitor:1969795 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 20:46:21,425 INFO HandlerThread:1969795 [system_monitor.py:probe():214] Collecting system info +2025-01-03 20:46:21,426 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 20:46:21,426 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started disk monitoring +2025-01-03 20:46:21,427 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 20:46:21,427 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started memory monitoring +2025-01-03 20:46:21,428 INFO SystemMonitor:1969795 [interfaces.py:start():188] Started network monitoring +2025-01-03 20:46:21,438 DEBUG HandlerThread:1969795 [system_info.py:probe():152] Probing system +2025-01-03 20:46:21,439 DEBUG HandlerThread:1969795 [system_info.py:_probe_git():137] Probing git +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:_probe_git():145] Probing git done +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:probe():200] Probing system done +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T01:46:21.438190', 'startedAt': '2025-01-04T01:46:21.074705', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.484375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.991, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 758.1241111755371}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 20:46:21,444 INFO HandlerThread:1969795 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 20:46:21,444 INFO HandlerThread:1969795 [system_monitor.py:probe():227] Publishing system info +2025-01-03 20:46:21,444 DEBUG HandlerThread:1969795 [system_info.py:_save_code():44] Saving code +2025-01-03 20:46:21,446 DEBUG HandlerThread:1969795 [system_info.py:_save_code():67] Saving code done +2025-01-03 20:46:21,446 DEBUG HandlerThread:1969795 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 20:46:21,508 DEBUG HandlerThread:1969795 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 20:46:21,509 INFO HandlerThread:1969795 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 20:46:21,512 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 20:46:21,512 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 20:46:21,747 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/qjr3og54-wandb-metadata.json +2025-01-03 20:46:21,755 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 20:46:21,755 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: python_packages +2025-01-03 20:46:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:21,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:21,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:21,759 INFO HandlerThread:1969795 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/tb" + save: true +} +control { + relay_id: "f961af653af74e62a8f20ce7de4ace91" +} +_info { + stream_id: "nibxxtai" +} + +2025-01-03 20:46:21,759 DEBUG HandlerThread:1969795 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 20:46:21,801 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/4yn8ctry-code/FISH/eval_robot.py +2025-01-03 20:46:21,814 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/a6a562hm-diff.patch +2025-01-03 20:46:21,863 DEBUG SenderThread:1969795 [sender.py:send():379] send: telemetry +2025-01-03 20:46:21,863 DEBUG SenderThread:1969795 [sender.py:send():379] send: tbrecord +2025-01-03 20:46:21,864 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:46:21,864 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 with policy live +2025-01-03 20:46:22,041 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ilo1kr6v-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code +2025-01-03 20:46:22,329 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH +2025-01-03 20:46:22,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:23,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:24,328 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:24,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:25,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:26,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:26,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:27,347 DEBUG SenderThread:1969795 [sender.py:send():379] send: config +2025-01-03 20:46:27,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:28,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:29,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:30,330 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:30,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:31,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:31,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:32,330 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:32,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:33,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:34,331 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:34,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:35,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:36,331 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:36,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:36,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:36,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:36,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:37,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:38,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:39,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:40,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:41,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:42,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:42,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:43,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:44,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:45,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:46,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:47,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:47,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:48,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:49,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:50,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:51,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:46:51,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:46:51,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:52,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:52,853 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:53,336 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:46:53,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:54,337 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:54,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:55,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:56,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:57,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:58,338 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:46:58,758 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:46:58,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:46:59,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:00,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:01,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:02,339 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:02,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:03,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:04,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:04,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:05,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:06,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:06,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:06,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:07,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:08,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:09,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:09,914 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:10,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:11,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:12,763 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:13,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:14,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:15,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:15,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:16,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:17,764 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:18,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:19,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:20,343 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:20,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:21,428 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 20:47:21,429 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:47:21,429 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:21,756 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:21,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:22,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:23,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:24,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:25,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:26,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:26,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:27,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:28,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:29,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:30,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:31,743 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:47:31,743 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 with policy now +2025-01-03 20:47:31,743 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:47:31,744 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:47:31,744 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:47:31,744 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:47:31,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:31,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:31,969 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/aj97t357-media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,346 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:47:32,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos +2025-01-03 20:47:32,628 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ojk5b6dv-media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:47:32,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:33,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:34,347 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:34,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:35,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:36,348 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:36,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:36,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:36,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:36,894 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:37,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:38,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:39,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:40,349 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:40,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:41,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:42,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:42,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:43,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:44,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:45,351 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:47:45,579 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/fjkpnicu-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:47:45,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:46,351 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:46,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:47,769 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:48,352 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:48,457 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:48,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:49,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:50,352 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:47:50,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:51,429 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:47:51,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:47:51,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:47:51,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:52,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:53,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:53,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:54,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:55,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:56,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:57,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:58,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:47:59,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:47:59,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:00,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:01,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:02,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:03,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:04,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:04,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:05,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:06,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:06,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:06,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:07,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:08,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:09,772 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:09,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:10,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:11,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:12,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:13,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:14,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:15,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:15,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:16,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:17,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:18,773 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:19,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:20,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:20,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:21,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:48:21,756 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:21,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:21,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:22,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:23,775 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:24,357 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:48:24,357 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 with policy now +2025-01-03 20:48:24,357 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:48:24,358 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:48:24,358 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:48:24,360 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:24,363 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:48:24,627 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/4zwkjt5i-media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:24,774 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:24,885 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/wduc7cuc-media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:48:25,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:26,364 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:26,378 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:26,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:27,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:28,776 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:29,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:30,365 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:30,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:31,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:31,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:32,366 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:32,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:33,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:34,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:35,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:36,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:36,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:36,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:36,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:37,367 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:48:37,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:38,191 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:48:38,191 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:48:38,191 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:48:38,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:48:38,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:38,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:39,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:40,368 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:40,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:41,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:42,369 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:48:42,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:42,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:43,778 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:44,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:45,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:46,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:47,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:47,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:48,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:49,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:50,779 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:51,430 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:48:51,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:48:51,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:48:51,797 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:52,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:52,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:53,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:54,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:55,780 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:56,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:57,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:58,761 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:48:58,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:48:59,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:00,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:01,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:02,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:03,781 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:04,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:04,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:05,790 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:06,757 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:06,757 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:06,801 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:07,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:08,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:09,792 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:09,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:10,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:11,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:12,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:13,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:14,379 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:14,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:15,509 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:15,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:16,380 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:16,798 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:17,277 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:49:17,277 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 with policy now +2025-01-03 20:49:17,278 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:49:17,278 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:49:17,279 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:49:17,282 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:49:17,380 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:49:17,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:49:17,492 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/7f8w5him-media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,748 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/kyniqsxz-media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:49:17,800 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:18,381 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:18,802 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:19,804 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:20,807 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:21,118 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:21,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:49:21,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:21,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:21,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:22,382 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:22,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:23,808 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:24,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:25,809 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:26,384 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:49:26,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:26,811 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:27,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:28,817 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:29,815 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:30,818 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:31,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:31,823 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:32,826 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:33,819 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:34,822 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:35,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:36,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:36,765 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:36,823 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:36,919 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:37,824 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:38,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:39,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:40,824 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:41,825 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:42,767 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:42,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:43,827 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:44,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:45,828 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:46,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:47,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:48,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:48,830 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:49,837 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:50,833 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:51,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:49:51,765 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:49:51,765 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:49:51,834 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:52,838 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:53,837 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:53,910 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:54,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:55,840 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:56,841 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:57,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:58,841 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:49:59,768 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:49:59,842 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:00,845 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:01,073 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:50:01,073 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:50:01,074 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:50:01,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:50:01,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:50:01,635 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/gdtcxzwk-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:50:01,843 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:02,396 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:02,844 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:03,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:04,397 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:04,845 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:05,689 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:05,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:06,398 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:06,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:06,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:06,846 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:07,846 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:08,399 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:08,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:09,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:10,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:10,914 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:11,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:12,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:13,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:14,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:15,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:16,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:16,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:17,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:18,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:19,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:20,403 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:20,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:21,431 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:50:21,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:21,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:21,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:21,899 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:22,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:23,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:24,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:25,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:26,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:27,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:27,848 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:28,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:29,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:30,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:31,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:32,766 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:32,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:33,849 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:34,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:35,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:36,759 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:36,759 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:36,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:37,847 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:37,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:38,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:39,850 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:40,409 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:40,851 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:41,753 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:50:41,753 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 with policy now +2025-01-03 20:50:41,755 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:50:41,755 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:50:41,755 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:50:41,758 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:50:41,851 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:42,410 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:50:42,601 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/uby4ka6c-media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:42,859 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:42,865 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/8kzzppl9-media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:50:43,813 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:43,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:44,411 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:44,862 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:45,866 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:46,411 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:46,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:47,862 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:48,866 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:49,377 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:49,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:50,413 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:50:50,865 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:51,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:50:51,762 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:50:51,762 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:50:51,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:52,869 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:53,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:54,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:54,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:50:55,870 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:56,871 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:57,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:58,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:50:59,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:00,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:00,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:01,144 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:01,144 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:01,146 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:01,416 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:01,416 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:01,657 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/zw5r0y49-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:01,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:02,417 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:02,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:03,882 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:04,417 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:04,872 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:05,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:06,110 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:06,418 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:06,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:06,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:06,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:07,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:08,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:09,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:10,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:11,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:11,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:12,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:13,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:14,873 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:15,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:16,770 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:16,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:17,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:18,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:19,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:20,424 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:20,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:21,432 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:51:21,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:21,761 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:21,832 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:21,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:22,425 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:22,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:23,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:24,426 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:24,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:25,874 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:26,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:27,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:27,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:28,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:29,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:30,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:31,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:32,875 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:33,771 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:33,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:34,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:35,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:36,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:36,760 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:36,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:37,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:38,820 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:38,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:39,876 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:40,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:40,878 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:41,217 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:41,217 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 with policy now +2025-01-03 20:51:41,220 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:41,221 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:41,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:41,229 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval +2025-01-03 20:51:41,437 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/mhv620ni-media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,693 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/2c0sloev-media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:51:41,883 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:42,432 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:42,888 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:43,433 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:43,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:44,793 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:44,890 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:45,899 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:46,434 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:46,889 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:47,905 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:48,901 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:49,894 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:49,905 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:50,435 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:50,893 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:50,997 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:51:50,997 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:51:51,207 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table +2025-01-03 20:51:51,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval +2025-01-03 20:51:51,470 INFO wandb-upload_0:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp8oaaost9 +2025-01-03 20:51:51,564 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 20:51:51,572 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 20:51:51,613 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/647pa3dy-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:51,760 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 20:51:51,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:51,907 INFO SenderThread:1969795 [sender.py:send_request_log_artifact():1525] logged artifact run-nibxxtai-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjAyOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NDM0Mg==', 'latestArtifact': None}} +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:51,907 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:51,907 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,908 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,908 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:51,908 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,912 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send():379] send: telemetry +2025-01-03 20:51:51,913 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,913 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,914 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,914 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,914 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,914 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,915 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,915 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,915 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,915 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,916 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,916 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,916 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,917 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,917 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,917 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,917 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,918 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,918 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:51,918 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:51,918 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 20:51:52,143 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/wi43gvgz-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:51:52,285 INFO wandb-upload_2:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp1pcwp_2j +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:52,436 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:52,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:52,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval +2025-01-03 20:51:52,829 INFO SenderThread:1969795 [sender.py:send_request_log_artifact():1525] logged artifact run-nibxxtai-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjA0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY3NDM0OQ==', 'latestArtifact': None}} +2025-01-03 20:51:52,829 DEBUG SenderThread:1969795 [sender.py:send():379] send: files +2025-01-03 20:51:52,830 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json with policy now +2025-01-03 20:51:52,830 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:51:52,830 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:51:52,831 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:51:52,831 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: stop_status +2025-01-03 20:51:52,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:53,049 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/kkjskfeh-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:53,281 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/5exuhlr2-media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:51:53,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:51:53,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:54,437 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:51:54,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:54,963 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:51:55,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:56,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:57,891 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:58,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:51:59,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:00,777 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:00,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:01,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:02,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:03,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:04,892 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:05,625 DEBUG SenderThread:1969795 [sender.py:send():379] send: exit +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():588] handling runtime: 344 +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:05,626 INFO SenderThread:1969795 [sender.py:send_exit():594] send defer +2025-01-03 20:52:05,626 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,626 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 20:52:05,627 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 1 +2025-01-03 20:52:05,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 20:52:05,627 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 20:52:05,627 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 2 +2025-01-03 20:52:05,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 20:52:05,627 INFO HandlerThread:1969795 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 20:52:05,627 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 20:52:05,628 DEBUG SystemMonitor:1969795 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 20:52:05,628 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 20:52:05,628 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 20:52:05,682 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [interfaces.py:finish():200] Joined network monitor +2025-01-03 20:52:05,683 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,683 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 20:52:05,683 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 3 +2025-01-03 20:52:05,683 DEBUG SenderThread:1969795 [sender.py:send():379] send: stats +2025-01-03 20:52:05,683 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,683 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 20:52:05,684 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:05,684 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 20:52:05,684 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 4 +2025-01-03 20:52:05,684 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:05,684 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 20:52:06,442 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:06,442 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:12,213 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:12,213 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,214 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 20:52:12,214 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 5 +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,214 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 20:52:12,214 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,215 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,215 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 20:52:12,215 DEBUG SenderThread:1969795 [sender.py:send():379] send: history +2025-01-03 20:52:12,216 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: summary_record +2025-01-03 20:52:12,216 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:12,217 DEBUG SenderThread:1969795 [sender.py:send():379] send: summary +2025-01-03 20:52:12,220 INFO SenderThread:1969795 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 20:52:12,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 6 +2025-01-03 20:52:12,221 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,221 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 20:52:12,221 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,221 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 20:52:12,225 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 20:52:12,348 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 7 +2025-01-03 20:52:12,348 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:12,348 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 20:52:12,348 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:12,348 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 20:52:12,444 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:12,444 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:52:12,627 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:15,064 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 8 +2025-01-03 20:52:15,065 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:15,065 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:15,065 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 20:52:15,065 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:15,065 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 20:52:15,065 INFO SenderThread:1969795 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 20:52:15,068 INFO SenderThread:1969795 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 20:52:15,070 INFO SenderThread:1969795 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 20:52:15,084 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 9 +2025-01-03 20:52:15,084 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:15,084 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 20:52:15,084 DEBUG SenderThread:1969795 [sender.py:send():379] send: artifact +2025-01-03 20:52:15,446 INFO Thread-12 :1969795 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:15,502 INFO wandb-upload_2:1969795 [upload_job.py:push():85] Skipped uploading /tmp/tmpb1t3b0z7/wandb-job.json +2025-01-03 20:52:15,502 INFO wandb-upload_1:1969795 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaz_huuo5 +2025-01-03 20:52:15,597 INFO wandb-upload_0:1969795 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgtghzagk +2025-01-03 20:52:15,628 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNzg3MjQ4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNzg1NzYxOA==', 'versionIndex': 407}}} +2025-01-03 20:52:16,172 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 20:52:16,172 INFO SenderThread:1969795 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 20:52:16,446 INFO SenderThread:1969795 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml config.yaml +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json wandb-summary.json +2025-01-03 20:52:16,447 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:52:16,448 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt requirements.txt +2025-01-03 20:52:16,448 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-metadata.json wandb-metadata.json +2025-01-03 20:52:16,449 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log output.log +2025-01-03 20:52:16,449 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/diff.patch diff.patch +2025-01-03 20:52:16,450 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 20:52:16,451 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 media/videos/eval/2_eval_2_121e578fe04e49712301.mp4 +2025-01-03 20:52:16,452 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 media/videos/eval/1_eval_1_da4e7b679afe12f99a3d.mp4 +2025-01-03 20:52:16,455 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 media/videos/eval/3_eval_3_a3410e3f59909246de3a.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 media/videos/eval/4_eval_4_1d5d9702636bb8a8301f.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 media/videos/eval/0_eval_0_24fb1057b6de7362aaa7.mp4 +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json media/table/eval/total_success_rate_ci_6_e32f206babc2825ce217.table.json +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-03 20:52:16,456 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 10 +2025-01-03 20:52:16,456 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,456 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,456 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 20:52:16,456 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,457 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 20:52:16,457 INFO SenderThread:1969795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:52:16,618 INFO wandb-upload_0:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/wandb-summary.json +2025-01-03 20:52:16,628 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,629 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,676 INFO wandb-upload_1:1969795 [upload_job.py:push():130] Uploaded file /tmp/tmp4g77fvztwandb/ekl1azpc-events.out.tfevents.1735955181.leonmkim-ROG-Strix-G15CS-G15CS.1969621.0 +2025-01-03 20:52:16,678 INFO wandb-upload_4:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/output.log +2025-01-03 20:52:16,678 INFO wandb-upload_2:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/config.yaml +2025-01-03 20:52:16,679 INFO wandb-upload_3:1969795 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/files/requirements.txt +2025-01-03 20:52:16,879 INFO Thread-11 (_thread_body):1969795 [sender.py:transition_state():614] send defer: 11 +2025-01-03 20:52:16,880 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,880 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 20:52:16,880 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,880 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 20:52:16,880 INFO SenderThread:1969795 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 20:52:16,881 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 12 +2025-01-03 20:52:16,881 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,881 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 20:52:16,882 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,882 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 20:52:16,882 INFO SenderThread:1969795 [file_stream.py:finish():601] file stream finish called +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [file_stream.py:finish():605] file stream finish is done +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 13 +2025-01-03 20:52:16,948 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,948 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 20:52:16,948 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 20:52:16,948 INFO SenderThread:1969795 [sender.py:transition_state():614] send defer: 14 +2025-01-03 20:52:16,948 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: defer +2025-01-03 20:52:16,948 INFO HandlerThread:1969795 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send():379] send: final +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send():379] send: footer +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: defer +2025-01-03 20:52:16,949 INFO SenderThread:1969795 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 20:52:16,949 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,949 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 20:52:16,950 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 20:52:16,950 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: server_info +2025-01-03 20:52:16,951 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 20:52:17,000 DEBUG SenderThread:1969795 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 20:52:17,000 INFO MainThread:1969795 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 20:52:17,001 INFO MainThread:1969795 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 20:52:17,001 INFO MainThread:1969795 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 20:52:17,001 DEBUG HandlerThread:1969795 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 20:52:17,001 INFO HandlerThread:1969795 [handler.py:finish():882] shutting down handler +2025-01-03 20:52:17,950 INFO WriterThread:1969795 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb +2025-01-03 20:52:18,000 INFO SenderThread:1969795 [sender.py:finish():1615] shutting down sender +2025-01-03 20:52:18,001 INFO SenderThread:1969795 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 20:52:18,001 INFO SenderThread:1969795 [file_pusher.py:join():175] waiting for file pusher diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug.log b/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4a05699ecce95a7c72de19df4e59b5b5b2915054 --- /dev/null +++ b/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Configure stats pid to 1969621 +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/settings +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 20:46:21,076 INFO MainThread:1969621 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug.log +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/wandb/run-20250103_204621-nibxxtai/logs/debug-internal.log +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():569] calling init triggers +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai'} +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():619] starting backend +2025-01-03 20:46:21,077 INFO MainThread:1969621 [wandb_init.py:init():623] setting up manager +2025-01-03 20:46:21,083 INFO MainThread:1969621 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 20:46:21,085 INFO MainThread:1969621 [wandb_init.py:init():631] backend started and connected +2025-01-03 20:46:21,098 INFO MainThread:1969621 [wandb_init.py:init():720] updated telemetry +2025-01-03 20:46:21,104 INFO MainThread:1969621 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 20:46:21,341 INFO MainThread:1969621 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 20:46:21,414 INFO MainThread:1969621 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 20:46:21,414 INFO MainThread:1969621 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 20:46:21,756 INFO MainThread:1969621 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 20:46:21,757 INFO MainThread:1969621 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 20:46:21,758 INFO MainThread:1969621 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 20:46:21,758 INFO MainThread:1969621 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/nibxxtai/tb, True +2025-01-03 20:46:27,347 INFO MainThread:1969621 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-03 20:52:18,002 WARNING MsgRouterThr:1969621 [router.py:message_loop():77] message_loop has been closed diff --git a/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb b/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb new file mode 100644 index 0000000000000000000000000000000000000000..432fa38bcee30b2e3c02c2748498f76feca3321d Binary files /dev/null and b/nibxxtai/wandb/run-20250103_204621-nibxxtai/run-nibxxtai.wandb differ diff --git a/p7zqzp4h/.hydra/config.yaml b/p7zqzp4h/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12a5f49d6069f4468b80ea35b60bdf34e31f1584 --- /dev/null +++ b/p7zqzp4h/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1930_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/p7zqzp4h/.hydra/hydra.yaml b/p7zqzp4h/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01be0750c9201e28d44c9be9b829db2601a02d75 --- /dev/null +++ b/p7zqzp4h/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/p7zqzp4h/.hydra/overrides.yaml b/p7zqzp4h/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/p7zqzp4h/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/p7zqzp4h/episode_rosbags/aligned_depth_to_color_K.npy b/p7zqzp4h/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/p7zqzp4h/episode_rosbags/cam_tf_world.npy b/p7zqzp4h/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/p7zqzp4h/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/p7zqzp4h/episode_rosbags/color_K.npy b/p7zqzp4h/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/p7zqzp4h/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/p7zqzp4h/episode_rosbags/depth_K.npy b/p7zqzp4h/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/p7zqzp4h/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/p7zqzp4h/eval_robot.log b/p7zqzp4h/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..e7445895271b1a742f011036e38f195997d6f72f --- /dev/null +++ b/p7zqzp4h/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-03 23:04:47,012][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-03 23:04:47,018][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/p7zqzp4h/eval_video/0_eval.mp4 b/p7zqzp4h/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0ca837a2eff0e71a03649b424316bacf01fcdb8 --- /dev/null +++ b/p7zqzp4h/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e28e7a4dc9b13a8d99065c7b7425ef176f3544092e8df5bacc5a511512c64229 +size 1437653 diff --git a/p7zqzp4h/eval_video/1_eval.mp4 b/p7zqzp4h/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..663b4f679140cbb136fc96ae5b6cdec027011a07 --- /dev/null +++ b/p7zqzp4h/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd20e3575921aea286e64b0365e676229f64a0df5be439008f6d810a70d290d +size 1530401 diff --git a/p7zqzp4h/eval_video/2_eval.mp4 b/p7zqzp4h/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46694f616ad107c905b32f6b18c76238f111b6b4 --- /dev/null +++ b/p7zqzp4h/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06a1064193e8e78ad79d7607debd55aeef0cace64486d0495d24e6fa447a609f +size 1526716 diff --git a/p7zqzp4h/eval_video/3_eval.mp4 b/p7zqzp4h/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44bc9435f97c920ac2954ab039b32fddced416e0 --- /dev/null +++ b/p7zqzp4h/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:068501d0d982850a2ba6dcbf803475f39f4b3226269639a6e5e791afcc8b0a75 +size 1461453 diff --git a/p7zqzp4h/eval_video/4_eval.mp4 b/p7zqzp4h/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c935fbf07140a36e97729b946d4c06c43a56e6a --- /dev/null +++ b/p7zqzp4h/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a407fa118ffc8cb3c152bba68b3d27b9108ae8bbdc99f08779e0b80b8b7bcc +size 1554359 diff --git a/p7zqzp4h/tb/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 b/p7zqzp4h/tb/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 new file mode 100644 index 0000000000000000000000000000000000000000..c53e115e29bebaf6ded71d9c26721cd1ed6c36f7 --- /dev/null +++ b/p7zqzp4h/tb/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3bf37452c1ff55428eec85fa69da13b4233675f3d9b411f7e9c0c2d460b837 +size 6539 diff --git a/p7zqzp4h/wandb/debug-internal.log b/p7zqzp4h/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..901976f55b01e904f818c985bb8d64bbc3ba4249 --- /dev/null +++ b/p7zqzp4h/wandb/debug-internal.log @@ -0,0 +1,813 @@ +2025-01-03 23:04:52,911 INFO StreamThr :1992255 [internal.py:wandb_internal():85] W&B internal server running at pid: 1992255, started at: 2025-01-03 23:04:52.910735 +2025-01-03 23:04:52,912 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status +2025-01-03 23:04:52,924 INFO WriterThread:1992255 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb +2025-01-03 23:04:52,924 DEBUG SenderThread:1992255 [sender.py:send():379] send: header +2025-01-03 23:04:52,931 DEBUG SenderThread:1992255 [sender.py:send():379] send: run +2025-01-03 23:04:53,190 INFO SenderThread:1992255 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files +2025-01-03 23:04:53,190 INFO SenderThread:1992255 [sender.py:_start_run_threads():1188] run started: p7zqzp4h with start time 1735963492.911532 +2025-01-03 23:04:53,209 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 23:04:53,209 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: check_version +2025-01-03 23:04:53,297 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 23:04:53,305 DEBUG HandlerThread:1992255 [system_info.py:__init__():26] System info init +2025-01-03 23:04:53,306 DEBUG HandlerThread:1992255 [system_info.py:__init__():41] System info init done +2025-01-03 23:04:53,306 INFO HandlerThread:1992255 [system_monitor.py:start():194] Starting system monitor +2025-01-03 23:04:53,306 INFO SystemMonitor:1992255 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 23:04:53,306 INFO HandlerThread:1992255 [system_monitor.py:probe():214] Collecting system info +2025-01-03 23:04:53,308 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 23:04:53,308 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started disk monitoring +2025-01-03 23:04:53,309 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 23:04:53,309 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started memory monitoring +2025-01-03 23:04:53,310 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started network monitoring +2025-01-03 23:04:53,321 DEBUG HandlerThread:1992255 [system_info.py:probe():152] Probing system +2025-01-03 23:04:53,323 DEBUG HandlerThread:1992255 [system_info.py:_probe_git():137] Probing git +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:_probe_git():145] Probing git done +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:probe():200] Probing system done +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T04:04:53.321843', 'startedAt': '2025-01-04T04:04:52.901055', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.994375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.984, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 557.6547737121582}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 23:04:53,328 INFO HandlerThread:1992255 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 23:04:53,328 INFO HandlerThread:1992255 [system_monitor.py:probe():227] Publishing system info +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:_save_code():44] Saving code +2025-01-03 23:04:53,329 DEBUG HandlerThread:1992255 [system_info.py:_save_code():67] Saving code done +2025-01-03 23:04:53,330 DEBUG HandlerThread:1992255 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 23:04:53,393 DEBUG HandlerThread:1992255 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 23:04:53,393 INFO HandlerThread:1992255 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 23:04:53,396 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:04:53,396 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 23:04:53,397 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 23:04:53,397 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 23:04:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 23:04:53,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: python_packages +2025-01-03 23:04:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:04:53,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:04:53,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:53,643 INFO HandlerThread:1992255 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/tb" + save: true +} +control { + relay_id: "f6eaf81feeac469680b843d6fc8f2e47" +} +_info { + stream_id: "p7zqzp4h" +} + +2025-01-03 23:04:53,643 DEBUG HandlerThread:1992255 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 23:04:53,661 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/c4wrhcr2-wandb-metadata.json +2025-01-03 23:04:53,696 INFO wandb-upload_2:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/epfu390h-diff.patch +2025-01-03 23:04:53,697 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ssp6g8vv-code/FISH/eval_robot.py +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: telemetry +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: tbrecord +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:04:53,786 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 with policy live +2025-01-03 23:04:53,983 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q6awrbek-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:04:54,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch +2025-01-03 23:04:54,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH +2025-01-03 23:04:54,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:55,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:56,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:04:56,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:57,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:58,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:58,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:59,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:00,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:01,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:02,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:03,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:03,687 DEBUG SenderThread:1992255 [sender.py:send():379] send: config +2025-01-03 23:05:03,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:04,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:05,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:06,193 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:06,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:07,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:08,193 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:08,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:08,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:08,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:08,765 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:09,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:10,194 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:10,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:11,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:12,195 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:12,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:13,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:14,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:14,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:15,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:16,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:17,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:18,196 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:18,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:19,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:19,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:20,197 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:20,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:21,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:22,198 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:22,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:23,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:23,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:23,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:24,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:24,792 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:25,199 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:05:25,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:26,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:27,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:28,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:29,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:30,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:30,985 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:31,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:32,200 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:32,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:33,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:34,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:35,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:36,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:36,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:37,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:38,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:38,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:38,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:39,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:40,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:41,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:41,749 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:42,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:43,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:44,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:45,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:46,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:47,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:47,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:48,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:49,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:50,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:51,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:52,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:52,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:53,310 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 23:05:53,311 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:05:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:53,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:53,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:54,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:55,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:56,203 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:05:56,203 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 with policy now +2025-01-03 23:05:56,203 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:05:56,204 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:05:56,204 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:05:56,205 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:05:56,207 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:56,207 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos +2025-01-03 23:05:56,477 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/idc2bgli-media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:56,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:56,729 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ut6w1gfa-media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:57,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:58,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:58,212 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:58,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:59,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:00,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:01,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:02,210 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:02,652 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:03,573 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:03,652 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:04,210 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:04,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:05,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:06,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:07,211 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:07,462 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ky20wlhv-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:07,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:08,212 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:08,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:08,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:08,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:08,768 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:09,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:10,212 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:10,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:11,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:12,213 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:12,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:13,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:14,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:14,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:15,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:16,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:17,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:18,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:19,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:20,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:20,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:21,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:22,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:23,311 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:06:23,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:23,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:23,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:24,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:25,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:25,792 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:26,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:27,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:28,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:29,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:30,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:31,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:31,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:32,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:33,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:34,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:35,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:36,221 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:36,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:36,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:37,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:38,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:38,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:38,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:39,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:40,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:41,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:42,174 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:42,223 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:42,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:43,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:44,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:45,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:46,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:46,841 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:06:46,842 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 with policy now +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:06:46,855 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:06:47,100 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/1h593t9l-media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:06:47,549 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/vykjhdb4-media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:47,856 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:48,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:48,662 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:49,659 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:50,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:51,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:52,226 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:52,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:53,171 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:53,312 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:06:53,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:53,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:53,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:54,227 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:54,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:55,063 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:06:55,063 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:06:55,064 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:06:55,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:06:55,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:55,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:56,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:56,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:57,662 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:58,229 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:58,520 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:58,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:59,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:00,229 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:00,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:01,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:02,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:03,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:03,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:04,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:05,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:06,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:07,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:08,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:08,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:08,689 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:08,766 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:09,669 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:10,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:11,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:12,233 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:12,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:13,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:14,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:14,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:15,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:16,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:17,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:18,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:19,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:19,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:20,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:21,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:22,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:23,312 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:07:23,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:23,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:23,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:24,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:24,800 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:25,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:26,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:27,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:28,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:29,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:30,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:30,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:31,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:32,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:33,669 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:34,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:34,481 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:07:34,481 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 with policy now +2025-01-03 23:07:34,484 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:07:34,484 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:07:34,484 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:07:34,493 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:07:34,671 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:34,757 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/2tm615il-media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:07:35,551 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/0x9pwkgd-media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:36,242 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:36,505 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:36,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:37,242 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:37,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:38,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:38,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:38,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:39,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:40,243 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:40,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:41,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:41,913 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:42,244 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:42,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:43,674 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:44,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:45,679 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:46,674 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:47,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:47,678 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:48,680 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:49,676 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:50,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:51,683 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:52,247 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:07:52,489 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/rppievis-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:07:52,679 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:52,911 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:07:52,911 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:07:52,911 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:52,912 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:07:53,247 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:07:53,313 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:07:53,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:53,649 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:53,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:54,248 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:54,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:55,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:56,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:57,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:58,249 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:58,257 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:58,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:59,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:00,683 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:01,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:02,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:03,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:03,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:04,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:05,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:06,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:07,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:08,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:08,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:08,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:08,756 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:09,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:10,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:11,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:12,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:13,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:14,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:14,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:15,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:16,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:17,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:18,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:19,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:19,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:20,257 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:20,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:21,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:22,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:23,313 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:08:23,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:23,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:23,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:24,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:24,758 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:25,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:26,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:27,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:28,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:29,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:30,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:30,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:31,689 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:32,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:33,199 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:08:33,199 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 with policy now +2025-01-03 23:08:33,203 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:08:33,203 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:08:33,203 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:08:33,216 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:08:33,473 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/kur6e580-media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:33,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:33,733 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ea90lg9n-media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:34,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:34,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:35,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:35,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:36,695 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:37,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:38,264 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:38,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:38,647 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:38,709 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:39,693 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:40,693 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:40,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:41,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:42,265 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:42,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:43,699 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:44,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:45,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:45,969 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:08:45,970 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:08:45,970 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:45,971 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:08:46,266 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:08:46,266 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:08:46,267 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:46,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:47,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:48,267 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:48,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:49,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:50,268 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:50,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:51,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:51,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:52,268 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:52,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:53,314 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:08:53,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:53,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:53,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:54,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:55,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:56,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:56,742 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:57,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:58,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:59,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:00,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:01,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:02,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:02,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:03,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:04,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:05,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:06,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:07,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:08,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:08,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:08,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:08,716 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:09,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:10,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:11,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:12,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:13,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:14,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:14,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:15,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:16,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:17,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:18,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:19,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:20,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:20,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:21,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:22,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:23,314 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:09:23,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:23,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:23,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:24,717 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:25,501 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:25,501 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 with policy now +2025-01-03 23:09:25,504 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:25,504 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:25,505 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:25,525 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:25,716 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:25,769 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q7d26e6b-media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,280 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:26,280 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:26,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:09:26,556 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:26,567 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/vlqcyvmb-media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,726 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:27,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:27,725 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:28,722 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:29,730 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:30,282 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:30,728 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:31,584 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:31,726 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:32,282 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:32,729 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:33,728 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:34,283 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:34,731 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:35,734 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:36,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:36,740 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:37,743 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:38,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:38,649 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:38,739 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:39,740 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:40,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:41,743 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:41,766 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:42,755 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:43,754 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:44,748 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:45,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:46,760 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:47,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:47,760 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:48,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:49,047 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:49,047 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,048 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,077 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:09:49,077 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media +2025-01-03 23:09:49,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval +2025-01-03 23:09:49,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table +2025-01-03 23:09:49,294 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:49,514 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q8to6zcj-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:49,529 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/71teo04v-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:49,608 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd0l7mqaf +2025-01-03 23:09:49,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:09:49,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:49,751 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:49,990 INFO SenderThread:1992255 [sender.py:send_request_log_artifact():1525] logged artifact run-p7zqzp4h-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTI1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5NDMxNg==', 'latestArtifact': None}} +2025-01-03 23:09:49,990 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:49,990 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 23:09:49,991 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:49,991 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,991 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,992 DEBUG SenderThread:1992255 [sender.py:send():379] send: telemetry +2025-01-03 23:09:49,992 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,995 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,996 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,996 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,996 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,996 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,997 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,997 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,997 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,997 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,998 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,998 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,998 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,998 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,999 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,999 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,000 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,000 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,000 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,000 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:09:50,215 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/728ogjmx-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:50,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval +2025-01-03 23:09:50,462 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ebqkuln5-media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:50,466 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptv9po9lm +2025-01-03 23:09:50,924 INFO SenderThread:1992255 [sender.py:send_request_log_artifact():1525] logged artifact run-p7zqzp4h-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTI3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5NDMxOA==', 'latestArtifact': None}} +2025-01-03 23:09:50,924 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:50,924 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json with policy now +2025-01-03 23:09:50,925 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:50,925 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,925 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,926 DEBUG SenderThread:1992255 [sender.py:send():379] send: exit +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():588] handling runtime: 296 +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():594] send defer +2025-01-03 23:09:50,926 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,926 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 23:09:50,927 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:50,928 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 23:09:50,928 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 1 +2025-01-03 23:09:50,928 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,928 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 23:09:50,929 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:50,929 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 23:09:50,929 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 2 +2025-01-03 23:09:50,929 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 23:09:50,929 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 23:09:50,929 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 23:09:50,930 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined network monitor +2025-01-03 23:09:51,007 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:51,007 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 23:09:51,007 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 3 +2025-01-03 23:09:51,008 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:09:51,008 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:51,008 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 23:09:51,008 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:51,008 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 23:09:51,008 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 4 +2025-01-03 23:09:51,008 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:51,008 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 23:09:51,156 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/fgooldr4-media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:51,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:52,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:57,050 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:57,051 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:57,051 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,052 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 23:09:57,052 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 5 +2025-01-03 23:09:57,052 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:57,052 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:57,053 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:57,053 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:57,054 DEBUG SenderThread:1992255 [sender.py:send():379] send: summary +2025-01-03 23:09:57,057 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:57,058 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 6 +2025-01-03 23:09:57,058 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,058 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 23:09:57,058 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 23:09:57,061 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:57,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:57,188 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 7 +2025-01-03 23:09:57,188 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:57,188 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,189 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 23:09:57,189 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,189 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 23:09:57,291 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:57,292 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:09:58,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:58,631 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 8 +2025-01-03 23:09:58,631 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:58,632 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:58,632 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 23:09:58,632 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:58,632 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 23:09:58,632 INFO SenderThread:1992255 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 23:09:58,634 INFO SenderThread:1992255 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 23:09:58,636 INFO SenderThread:1992255 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 23:09:58,655 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 9 +2025-01-03 23:09:58,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:58,655 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 23:09:58,655 DEBUG SenderThread:1992255 [sender.py:send():379] send: artifact +2025-01-03 23:09:59,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:59,141 INFO wandb-upload_1:1992255 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp87z3f_qi +2025-01-03 23:09:59,141 INFO wandb-upload_0:1992255 [upload_job.py:push():85] Skipped uploading /tmp/tmpzf6e1rpy/wandb-job.json +2025-01-03 23:09:59,238 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphrizh1qg +2025-01-03 23:09:59,292 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTQzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'versionIndex': 414}}} +2025-01-03 23:09:59,873 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml config.yaml +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json wandb-summary.json +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt requirements.txt +2025-01-03 23:10:00,296 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json wandb-metadata.json +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log output.log +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch diff.patch +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:10:00,300 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:10:00,302 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:10:00,302 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 10 +2025-01-03 23:10:00,302 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:00,302 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,302 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 23:10:00,303 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,303 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 23:10:00,303 INFO SenderThread:1992255 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:10:00,481 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:10:00,502 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:10:00,531 INFO wandb-upload_4:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:10:00,532 INFO wandb-upload_2:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/aycs9ny9-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:10:00,562 INFO wandb-upload_3:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt +2025-01-03 23:10:00,763 INFO Thread-11 (_thread_body):1992255 [sender.py:transition_state():614] send defer: 11 +2025-01-03 23:10:00,763 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,763 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 23:10:00,763 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,763 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 23:10:00,763 INFO SenderThread:1992255 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 12 +2025-01-03 23:10:00,765 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,765 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 23:10:00,765 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [file_stream.py:finish():601] file stream finish called +2025-01-03 23:10:00,951 INFO SenderThread:1992255 [file_stream.py:finish():605] file stream finish is done +2025-01-03 23:10:00,951 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 13 +2025-01-03 23:10:00,951 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,951 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 14 +2025-01-03 23:10:00,952 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,952 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send():379] send: final +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send():379] send: footer +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 23:10:00,953 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:10:00,953 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:00,953 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 23:10:00,954 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:10:00,954 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: server_info +2025-01-03 23:10:00,954 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 23:10:00,956 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 23:10:00,956 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:10:01,010 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 23:10:01,011 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 23:10:01,012 INFO HandlerThread:1992255 [handler.py:finish():882] shutting down handler +2025-01-03 23:10:01,954 INFO WriterThread:1992255 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [sender.py:finish():1615] shutting down sender +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [file_pusher.py:join():175] waiting for file pusher diff --git a/p7zqzp4h/wandb/debug.log b/p7zqzp4h/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..56693f17fdcbdc6cff5cfc6610279f821ee1c06d --- /dev/null +++ b/p7zqzp4h/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Configure stats pid to 1992075 +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/settings +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug.log +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug-internal.log +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():569] calling init triggers +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h'} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():619] starting backend +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():623] setting up manager +2025-01-03 23:04:52,910 INFO MainThread:1992075 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 23:04:52,911 INFO MainThread:1992075 [wandb_init.py:init():631] backend started and connected +2025-01-03 23:04:52,923 INFO MainThread:1992075 [wandb_init.py:init():720] updated telemetry +2025-01-03 23:04:52,929 INFO MainThread:1992075 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 23:04:53,209 INFO MainThread:1992075 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 23:04:53,247 INFO MainThread:1992075 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 23:04:53,247 INFO MainThread:1992075 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 23:04:53,642 INFO MainThread:1992075 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 23:04:53,642 INFO MainThread:1992075 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/tb, True +2025-01-03 23:05:03,687 INFO MainThread:1992075 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 23:10:02,012 WARNING MsgRouterThr:1992075 [router.py:message_loop():77] message_loop has been closed diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33f511b32bb8801566b0360501a2ca5350928c9c --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1930_0' +wandb_notes: + desc: null + value: 1930_0_restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735963492 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f345052a4098779e6d1835354dda24725cd2d7e6 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 57a851f..7b7e871 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -138,12 +138,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' ++# wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1006_0' ++# wandb_run_id: '1926_1' # seed 1 ++wandb_run_id: '1930_0' # dataset shuffle seed 1 ++ ++ + + # RGBD+mask+contact(contact+DTC)+act history +-wandb_run_id: '1745_0' ++# wandb_run_id: '1745_0' + + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 71a5d1c..c7eadce 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -159,8 +159,10 @@ run_id_list = [ + ] + + run_id_list = [ +- # '1006_0', +- '1745_0' ++ # '1922_1', ++ # '1922_2', ++ '1926_1', ++ '1930_0', + ] + + checkpoint_epoch = 12000 +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 6f97e26..285afc8 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -330,8 +330,8 @@ class Franka: + # self.yaw = yaw + + self.move_to_target_hz = 10.0 +- self.move_to_trans_tolerance = 0.005 +- self.move_to_rot_tolerance = 1.0 ++ self.move_to_trans_tolerance = 0.01 ++ self.move_to_rot_tolerance = 2.0 + self.move_to_target_max_velocity = 0.8 + self.move_to_target_max_distance = self.move_to_target_max_velocity * (1/self.move_to_target_hz) + self.move_to_target_timeout = 5.0 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..2f85f61c92088a8993fdf1bc1b36967832fa135d --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.25744009022795533, 0.9953789374731651, 0.284928768916229, 0.987118244073719, 0.4244327843340374, 0.9746108054165524, 0.6104677766340574, 0.9433402008421657]]} \ No newline at end of file diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..5e9adaa506f5e0f85d4a7a60a8344ac933eaef32 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[21, 25, 0.84, 0.6140734851132175, 0.9605799465450376, 0.6677999645135972, 0.9539160206006116, 0.6978200673661368, 0.929663080636161, 0.7128638214937297, 0.9258960074824649]]} \ No newline at end of file diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a0ca837a2eff0e71a03649b424316bacf01fcdb8 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e28e7a4dc9b13a8d99065c7b7425ef176f3544092e8df5bacc5a511512c64229 +size 1437653 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..663b4f679140cbb136fc96ae5b6cdec027011a07 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd20e3575921aea286e64b0365e676229f64a0df5be439008f6d810a70d290d +size 1530401 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46694f616ad107c905b32f6b18c76238f111b6b4 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06a1064193e8e78ad79d7607debd55aeef0cace64486d0495d24e6fa447a609f +size 1526716 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44bc9435f97c920ac2954ab039b32fddced416e0 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:068501d0d982850a2ba6dcbf803475f39f4b3226269639a6e5e791afcc8b0a75 +size 1461453 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5c935fbf07140a36e97729b946d4c06c43a56e6a --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a407fa118ffc8cb3c152bba68b3d27b9108ae8bbdc99f08779e0b80b8b7bcc +size 1554359 diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a1516717d9cee994004efab8399575cc94fa1e93 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735963503.687824]: resetting environment +[INFO] [1735963503.692587]: cleared current plan +[INFO] [1735963503.693460]: moving to home +[INFO] [1735963504.695824]: reached home +[INFO] [1735963504.697351]: reset action history +[INFO] [1735963507.409081]: environment reset +Starting episode 0 +[INFO] [1735963507.413786]: resetting environment +[INFO] [1735963507.418758]: cleared current plan +[INFO] [1735963507.419608]: moving to home +[INFO] [1735963508.421268]: reached home +[INFO] [1735963508.422560]: reset action history +[INFO] [1735963511.133654]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735963516.311305]: resetting environment +[INFO] [1735963516.318492]: cleared current plan +[INFO] [1735963516.319501]: moving to home +[INFO] [1735963517.321260]: reached home +[INFO] [1735963517.322760]: reset action history +[INFO] [1735963520.031086]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/episode_rosbags/episode_0_2025-01-03-23-05-20.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1992075.log +started roslaunch server http://158.130.50.37:35351/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1992364] +process[print_text-2]: started with pid [1992388] +process[rosbag_record-3]: started with pid [1992389] +started bagging! +For topic gripper_width: timestamp difference is 105541264 for nearest: 1735963529928197657 - target: 1735963529822656393 at idx 125 +For topic gripper_width: timestamp difference is -107795474 for nearest: 1735963554561511519 - target: 1735963554669306993 at idx 249 +[INFO] [1735963555.252735]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735963555.985494]: Stored episode 1. +[INFO] [1735963555.985726]: Saving video... +[INFO] [1735963556.190213]: Video saved! +Starting episode 1 +[INFO] [1735963556.204149]: resetting environment +[INFO] [1735963556.210566]: cleared current plan +[INFO] [1735963556.211589]: moving to home +[INFO] [1735963559.814692]: reached home +[INFO] [1735963559.816973]: reset action history +[INFO] [1735963562.528076]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963567.002631]: resetting environment +[INFO] [1735963567.006267]: cleared current plan +[INFO] [1735963567.006438]: moving to home +[INFO] [1735963568.008291]: reached home +[INFO] [1735963568.008701]: reset action history +[INFO] [1735963570.718293]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/episode_rosbags/episode_1_2025-01-03-23-06-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1992075.log +started roslaunch server http://158.130.50.37:37541/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1992514] +process[print_text-5]: started with pid [1992515] +process[rosbag_record-6]: started with pid [1992516] +started bagging! +For topic gripper_width: timestamp difference is 124341095 for nearest: 1735963594628182257 - target: 1735963594503841162 at idx 249 +For topic gripper_width: timestamp difference is -170410414 for nearest: 1735963600861539583 - target: 1735963601031949997 at idx 249 +[INFO] [1735963605.623085]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1735963606.377170]: Stored episode 2. +[INFO] [1735963606.377411]: Saving video... +[INFO] [1735963606.797333]: Video saved! +Starting episode 2 +[INFO] [1735963606.819923]: resetting environment +[INFO] [1735963606.831659]: cleared current plan +[INFO] [1735963606.832013]: moving to home +[INFO] [1735963610.433854]: reached home +[INFO] [1735963610.436929]: reset action history +[INFO] [1735963613.169791]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963614.800548]: resetting environment +[INFO] [1735963614.809019]: cleared current plan +[INFO] [1735963614.809439]: moving to home +[INFO] [1735963615.810876]: reached home +[INFO] [1735963615.811769]: reset action history +[INFO] [1735963618.520280]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/episode_rosbags/episode_2_2025-01-03-23-06-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1992075.log +started roslaunch server http://158.130.50.37:45731/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1992655] +process[print_text-8]: started with pid [1992656] +process[rosbag_record-9]: started with pid [1992680] +started bagging! +For topic gripper_width: timestamp difference is 114149764 for nearest: 1735963629628208354 - target: 1735963629514058590 at idx 140 +[INFO] [1735963653.167840]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1735963654.159580]: Stored episode 3. +[INFO] [1735963654.160138]: Saving video... +[INFO] [1735963654.443391]: Video saved! +Starting episode 3 +[INFO] [1735963654.490580]: resetting environment +[INFO] [1735963654.502110]: cleared current plan +[INFO] [1735963654.504351]: moving to home +[WARN] [1735963654.497150]: Plan exhausted +[INFO] [1735963658.109539]: reached home +[INFO] [1735963658.130270]: reset action history +[INFO] [1735963660.868764]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963672.183645]: resetting environment +[INFO] [1735963672.198191]: cleared current plan +[INFO] [1735963672.198681]: moving to home +[INFO] [1735963673.200226]: reached home +[INFO] [1735963673.201886]: reset action history +[INFO] [1735963675.981688]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/episode_rosbags/episode_3_2025-01-03-23-07-55.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1992075.log +started roslaunch server http://158.130.50.37:35291/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1992829] +process[print_text-11]: started with pid [1992830] +process[rosbag_record-12]: started with pid [1992831] +started bagging! +For topic gripper_width: timestamp difference is 141703490 for nearest: 1735963698628176335 - target: 1735963698486472845 at idx 249 +[INFO] [1735963711.909827]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735963712.659480]: Stored episode 4. +[INFO] [1735963712.663870]: Saving video... +[INFO] [1735963713.134990]: Video saved! +Starting episode 4 +[INFO] [1735963713.208335]: resetting environment +[INFO] [1735963713.235718]: cleared current plan +[INFO] [1735963713.238752]: moving to home +[INFO] [1735963716.848266]: reached home +[INFO] [1735963716.855027]: reset action history +[INFO] [1735963719.646698]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1735963725.455982]: resetting environment +[INFO] [1735963725.496200]: cleared current plan +[INFO] [1735963725.501745]: moving to home +[INFO] [1735963726.506838]: reached home +[INFO] [1735963726.508388]: reset action history +[INFO] [1735963729.238934]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/episode_rosbags/episode_4_2025-01-03-23-08-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1992075.log +started roslaunch server http://158.130.50.37:39497/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1993036] +process[print_text-14]: started with pid [1993037] +process[rosbag_record-15]: started with pid [1993038] +started bagging! +[INFO] [1735963764.223537]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1735963765.008167]: Stored episode 5. +[INFO] [1735963765.019042]: Saving video... +[INFO] [1735963765.442434]: Video saved! +[INFO] [1735963765.505541]: resetting environment +[INFO] [1735963765.552917]: cleared current plan +[INFO] [1735963765.554207]: moving to home +[WARN] [1735963765.546794]: Plan exhausted +[INFO] [1735963768.761054]: reached home +[INFO] [1735963768.777891]: reset action history +[INFO] [1735963771.584155]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5558452ddc4037735c2eba67eeda2c7e17e68a0e --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-04T04:04:53.321843", + "startedAt": "2025-01-04T04:04:52.901055", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "c537eb6734155592809c2a7db5cca971ec186d72" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3374.994375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.984, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 557.6547737121582 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..32bcd91573da9b644888eb2618db8da05fad4719 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "e28e7a4dc9b13a8d99065c7b7425ef176f3544092e8df5bacc5a511512c64229", "size": 1437653, "path": "media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4"}, "global_step": 665, "_timestamp": 1735963789.5773387, "_runtime": 296.6658067703247, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "7cd20e3575921aea286e64b0365e676229f64a0df5be439008f6d810a70d290d", "size": 1530401, "path": "media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4"}, "eval/num_success": 4.0, "episode": 25.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.25744009017944336, "eval/success_rate_ub_0.01": 0.9953789114952087, "eval/success_rate_lb_0.025": 0.2849287688732147, "eval/success_rate_ub_0.025": 0.9871182441711426, "eval/success_rate_lb_0.05": 0.42443278431892395, "eval/success_rate_ub_0.05": 0.9746108055114746, "eval/success_rate_lb_0.1": 0.610467791557312, "eval/success_rate_ub_0.1": 0.9433401823043823, "eval/2_eval": {"_type": "video-file", "sha256": "06a1064193e8e78ad79d7607debd55aeef0cace64486d0495d24e6fa447a609f", "size": 1526716, "path": "media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "068501d0d982850a2ba6dcbf803475f39f4b3226269639a6e5e791afcc8b0a75", "size": 1461453, "path": "media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "03a407fa118ffc8cb3c152bba68b3d27b9108ae8bbdc99f08779e0b80b8b7bcc", "size": 1554359, "path": "media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "f6f07faa4a586efd9baa9e785d4017c5140aeeafdfeb020c33b4bb37a7d266af", "size": 397, "artifact_path": "wandb-client-artifact://ebqc81zif4fgnoqac87vy3mg29b0gzkrl56ujq49jhk1x3z5zsrincj61niybxcydbigw31rlprcqwi0rnmw3otc343lr7trtpzuffve8f5wib600een4o58ykav3ke4/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://jz5np1xc4lraaiqv74bruemho2yq2hen01jyl92h8e4zjknur7rc8aq4pa5ikntcy76zyv1y4kpjubwztol3tpzrb9zgox1m2g500ferl3pelifybyhpblsl6j8xz3ic:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 21, "total_num_episodes": 25, "total_success_rate": 0.84, "total_success_rate_lb_0.01": 0.6140734851132175, "total_success_rate_ub_0.01": 0.9605799465450376, "total_success_rate_lb_0.025": 0.6677999645135972, "total_success_rate_ub_0.025": 0.9539160206006116, "total_success_rate_lb_0.05": 0.6978200673661368, "total_success_rate_ub_0.05": 0.929663080636161, "total_success_rate_lb_0.1": 0.7128638214937297, "total_success_rate_ub_0.1": 0.9258960074824649, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "412f47f3cc747f52add46e34137c086ae8abc2145e146b48d2a6437dc8bf0e8a", "size": 505, "artifact_path": "wandb-client-artifact://3xsgqku7qnnjxrxeir7yrgbae7j2dhx3pj57v5em8rfcpwthgbai8mzbaz07du1ak7o7twb1x69esiwyjlmax34cnpy6r1hvkxogasm80ie4n0rakyfckepl68twuyim/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ao0qfcynwnes0hecilwnnwhsgbzs2pnvdgojfii80eay7mioe7mto1m187teg1jz0zxw22ikw49bieym402sq3b0vaxnoqrpcjv8o6wjdf6rfd5xs39a8gfmwngyl4t3:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 296}, "eval/total_success_rate": 0.8399999737739563, "eval/total_success_rate_lb_0.01": 0.6140734553337097, "eval/total_success_rate_ub_0.01": 0.9605799317359924, "eval/total_success_rate_lb_0.025": 0.6677999496459961, "eval/total_success_rate_ub_0.025": 0.9539160132408142, "eval/total_success_rate_lb_0.05": 0.6978200674057007, "eval/total_success_rate_ub_0.05": 0.9296630620956421, "eval/total_success_rate_lb_0.1": 0.7128638029098511, "eval/total_success_rate_ub_0.1": 0.9258959889411926} \ No newline at end of file diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug-internal.log b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..901976f55b01e904f818c985bb8d64bbc3ba4249 --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug-internal.log @@ -0,0 +1,813 @@ +2025-01-03 23:04:52,911 INFO StreamThr :1992255 [internal.py:wandb_internal():85] W&B internal server running at pid: 1992255, started at: 2025-01-03 23:04:52.910735 +2025-01-03 23:04:52,912 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status +2025-01-03 23:04:52,924 INFO WriterThread:1992255 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb +2025-01-03 23:04:52,924 DEBUG SenderThread:1992255 [sender.py:send():379] send: header +2025-01-03 23:04:52,931 DEBUG SenderThread:1992255 [sender.py:send():379] send: run +2025-01-03 23:04:53,190 INFO SenderThread:1992255 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files +2025-01-03 23:04:53,190 INFO SenderThread:1992255 [sender.py:_start_run_threads():1188] run started: p7zqzp4h with start time 1735963492.911532 +2025-01-03 23:04:53,209 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: check_version +2025-01-03 23:04:53,209 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: check_version +2025-01-03 23:04:53,297 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: run_start +2025-01-03 23:04:53,305 DEBUG HandlerThread:1992255 [system_info.py:__init__():26] System info init +2025-01-03 23:04:53,306 DEBUG HandlerThread:1992255 [system_info.py:__init__():41] System info init done +2025-01-03 23:04:53,306 INFO HandlerThread:1992255 [system_monitor.py:start():194] Starting system monitor +2025-01-03 23:04:53,306 INFO SystemMonitor:1992255 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-03 23:04:53,306 INFO HandlerThread:1992255 [system_monitor.py:probe():214] Collecting system info +2025-01-03 23:04:53,308 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started cpu monitoring +2025-01-03 23:04:53,308 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started disk monitoring +2025-01-03 23:04:53,309 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started gpu monitoring +2025-01-03 23:04:53,309 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started memory monitoring +2025-01-03 23:04:53,310 INFO SystemMonitor:1992255 [interfaces.py:start():188] Started network monitoring +2025-01-03 23:04:53,321 DEBUG HandlerThread:1992255 [system_info.py:probe():152] Probing system +2025-01-03 23:04:53,323 DEBUG HandlerThread:1992255 [system_info.py:_probe_git():137] Probing git +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:_probe_git():145] Probing git done +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:probe():200] Probing system done +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-04T04:04:53.321843', 'startedAt': '2025-01-04T04:04:52.901055', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'c537eb6734155592809c2a7db5cca971ec186d72'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3374.994375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.984, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 557.6547737121582}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-03 23:04:53,328 INFO HandlerThread:1992255 [system_monitor.py:probe():224] Finished collecting system info +2025-01-03 23:04:53,328 INFO HandlerThread:1992255 [system_monitor.py:probe():227] Publishing system info +2025-01-03 23:04:53,328 DEBUG HandlerThread:1992255 [system_info.py:_save_code():44] Saving code +2025-01-03 23:04:53,329 DEBUG HandlerThread:1992255 [system_info.py:_save_code():67] Saving code done +2025-01-03 23:04:53,330 DEBUG HandlerThread:1992255 [system_info.py:_save_patches():84] Saving git patches +2025-01-03 23:04:53,393 DEBUG HandlerThread:1992255 [system_info.py:_save_patches():126] Saving git patches done +2025-01-03 23:04:53,393 INFO HandlerThread:1992255 [system_monitor.py:probe():229] Finished publishing system info +2025-01-03 23:04:53,396 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:04:53,396 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-03 23:04:53,397 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-03 23:04:53,397 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-03 23:04:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: python_packages +2025-01-03 23:04:53,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: python_packages +2025-01-03 23:04:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:04:53,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:04:53,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:53,643 INFO HandlerThread:1992255 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/tb" + save: true +} +control { + relay_id: "f6eaf81feeac469680b843d6fc8f2e47" +} +_info { + stream_id: "p7zqzp4h" +} + +2025-01-03 23:04:53,643 DEBUG HandlerThread:1992255 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-03 23:04:53,661 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/c4wrhcr2-wandb-metadata.json +2025-01-03 23:04:53,696 INFO wandb-upload_2:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/epfu390h-diff.patch +2025-01-03 23:04:53,697 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ssp6g8vv-code/FISH/eval_robot.py +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: telemetry +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: tbrecord +2025-01-03 23:04:53,786 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:04:53,786 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 with policy live +2025-01-03 23:04:53,983 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q6awrbek-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:04:54,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch +2025-01-03 23:04:54,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code +2025-01-03 23:04:54,192 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH +2025-01-03 23:04:54,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:55,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:56,191 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:04:56,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:57,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:58,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:04:58,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:04:59,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:00,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:01,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:02,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:03,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:03,687 DEBUG SenderThread:1992255 [sender.py:send():379] send: config +2025-01-03 23:05:03,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:04,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:05,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:06,193 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:06,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:07,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:08,193 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:08,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:08,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:08,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:08,765 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:09,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:10,194 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:10,642 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:11,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:12,195 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:12,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:13,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:14,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:14,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:15,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:16,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:17,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:18,196 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:18,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:19,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:19,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:20,197 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:20,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:21,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:22,198 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:22,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:23,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:23,640 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:23,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:24,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:24,792 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:25,199 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:05:25,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:26,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:27,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:28,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:29,646 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:30,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:30,985 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:31,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:32,200 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:32,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:33,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:34,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:35,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:36,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:36,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:37,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:38,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:38,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:38,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:39,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:40,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:41,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:41,749 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:42,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:43,648 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:44,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:45,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:46,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:47,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:47,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:48,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:49,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:50,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:51,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:52,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:52,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:53,310 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-03 23:05:53,311 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:05:53,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:05:53,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:05:53,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:54,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:55,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:56,203 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:05:56,203 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 with policy now +2025-01-03 23:05:56,203 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:05:56,204 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:05:56,204 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:05:56,205 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:05:56,207 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:56,207 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:05:56,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos +2025-01-03 23:05:56,477 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/idc2bgli-media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:56,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:56,729 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ut6w1gfa-media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:05:57,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:58,208 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:05:58,212 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:05:58,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:05:59,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:00,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:01,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:02,210 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:02,652 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:03,573 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:03,652 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:04,210 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:04,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:05,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:06,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:07,211 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:07,462 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ky20wlhv-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:07,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:08,212 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:08,640 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:08,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:08,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:08,768 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:09,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:10,212 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:10,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:11,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:12,213 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:12,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:13,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:14,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:14,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:15,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:16,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:17,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:18,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:19,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:20,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:20,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:21,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:22,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:23,311 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:06:23,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:23,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:23,681 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:24,654 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:25,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:25,792 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:26,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:27,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:28,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:29,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:30,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:31,645 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:31,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:32,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:33,656 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:34,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:35,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:36,221 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:36,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:36,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:37,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:38,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:38,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:38,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:39,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:40,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:41,657 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:42,174 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:42,223 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:42,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:43,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:44,658 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:45,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:46,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:46,841 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:06:46,842 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 with policy now +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:06:46,842 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:06:46,855 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:06:47,100 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/1h593t9l-media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:06:47,549 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/vykjhdb4-media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:06:47,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:47,856 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:48,225 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:48,662 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:49,659 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:50,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:51,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:52,226 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:52,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:53,171 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:53,312 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:06:53,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:06:53,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:06:53,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:54,227 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:54,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:55,063 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:06:55,063 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:06:55,064 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:06:55,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:06:55,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:06:55,661 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:56,228 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:56,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:57,662 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:58,229 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:06:58,520 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:06:58,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:06:59,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:00,229 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:00,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:01,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:02,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:03,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:03,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:04,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:05,664 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:06,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:07,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:08,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:08,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:08,689 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:08,766 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:09,669 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:10,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:11,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:12,233 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:12,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:13,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:14,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:14,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:15,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:16,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:17,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:18,666 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:19,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:19,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:20,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:21,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:22,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:23,312 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:07:23,641 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:23,641 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:23,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:24,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:24,800 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:25,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:26,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:27,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:28,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:29,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:30,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:30,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:31,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:32,668 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:33,669 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:34,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:34,481 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:07:34,481 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 with policy now +2025-01-03 23:07:34,484 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:07:34,484 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:07:34,484 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:07:34,493 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:07:34,671 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:34,757 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/2tm615il-media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,241 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:07:35,551 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/0x9pwkgd-media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:07:35,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:36,242 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:36,505 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:36,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:37,242 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:37,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:38,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:38,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:38,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:39,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:40,243 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:40,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:41,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:41,913 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:42,244 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:42,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:43,674 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:44,675 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:45,679 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:46,674 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:47,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:47,678 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:48,680 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:49,676 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:50,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:51,683 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:52,247 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:07:52,489 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/rppievis-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:07:52,679 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:52,911 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:07:52,911 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:07:52,911 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:52,912 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:07:53,247 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:07:53,313 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:07:53,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:07:53,649 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:07:53,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:54,248 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:54,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:55,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:56,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:57,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:58,249 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:07:58,257 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:07:58,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:07:59,682 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:00,683 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:01,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:02,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:03,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:03,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:04,684 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:05,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:06,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:07,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:08,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:08,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:08,685 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:08,756 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:09,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:10,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:11,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:12,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:13,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:14,650 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:14,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:15,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:16,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:17,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:18,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:19,673 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:19,686 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:20,257 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:20,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:21,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:22,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:23,313 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:08:23,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:23,643 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:23,687 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:24,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:24,758 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:25,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:26,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:27,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:28,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:29,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:30,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:30,688 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:31,689 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:32,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:33,199 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:08:33,199 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 with policy now +2025-01-03 23:08:33,203 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:08:33,203 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:08:33,203 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:08:33,216 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:33,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:08:33,473 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/kur6e580-media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:33,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:33,733 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ea90lg9n-media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:08:34,262 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:34,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:35,651 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:35,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:36,695 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:37,694 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:38,264 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:38,647 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:38,647 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:38,709 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:39,693 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:40,693 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:40,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:41,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:42,265 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:42,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:43,699 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:44,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:45,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:45,969 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:08:45,970 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:08:45,970 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:45,971 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:08:46,266 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:08:46,266 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:08:46,267 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:46,701 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:47,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:48,267 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:48,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:49,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:50,268 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:50,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:51,690 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:51,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:52,268 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:08:52,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:53,314 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:08:53,643 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:08:53,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:08:53,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:54,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:55,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:56,702 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:56,742 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:08:57,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:58,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:08:59,703 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:00,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:01,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:02,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:02,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:03,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:04,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:05,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:06,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:07,704 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:08,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:08,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:08,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:08,716 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:09,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:10,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:11,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:12,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:13,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:14,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:14,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:15,705 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:16,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:17,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:18,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:19,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:20,653 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:20,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:21,706 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:22,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:23,314 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:09:23,644 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:23,644 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:23,707 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:24,717 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:25,501 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:25,501 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 with policy now +2025-01-03 23:09:25,504 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:25,504 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:25,505 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:25,525 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:25,716 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:25,769 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q7d26e6b-media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,280 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:26,280 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:26,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval +2025-01-03 23:09:26,556 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:26,567 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/vlqcyvmb-media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:09:26,726 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:27,281 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:27,725 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:28,722 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:29,730 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:30,282 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:30,728 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:31,584 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:31,726 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:32,282 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:32,729 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:33,728 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:34,283 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:34,731 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:35,734 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:36,667 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:36,740 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:37,743 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:38,649 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: stop_status +2025-01-03 23:09:38,649 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: stop_status +2025-01-03 23:09:38,739 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:39,740 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:40,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:41,743 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:41,766 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:42,755 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:43,754 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:44,748 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:45,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:46,760 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:47,665 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:47,760 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:48,753 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:49,047 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:49,047 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,048 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,077 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:09:49,077 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:49,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media +2025-01-03 23:09:49,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval +2025-01-03 23:09:49,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table +2025-01-03 23:09:49,294 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:49,514 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/q8to6zcj-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:49,529 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/71teo04v-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:49,608 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpd0l7mqaf +2025-01-03 23:09:49,663 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-03 23:09:49,672 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: partial_history +2025-01-03 23:09:49,751 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:09:49,990 INFO SenderThread:1992255 [sender.py:send_request_log_artifact():1525] logged artifact run-p7zqzp4h-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTI1OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5NDMxNg==', 'latestArtifact': None}} +2025-01-03 23:09:49,990 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:49,990 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json with policy now +2025-01-03 23:09:49,991 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:49,991 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,991 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,992 DEBUG SenderThread:1992255 [sender.py:send():379] send: telemetry +2025-01-03 23:09:49,992 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,995 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,996 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,996 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,996 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,996 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,997 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,997 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,997 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,997 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,998 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,998 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,998 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,998 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,999 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:49,999 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:49,999 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,000 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,000 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,000 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,000 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: log_artifact +2025-01-03 23:09:50,215 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/728ogjmx-media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:09:50,288 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:50,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval +2025-01-03 23:09:50,462 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/ebqkuln5-media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:50,466 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptv9po9lm +2025-01-03 23:09:50,924 INFO SenderThread:1992255 [sender.py:send_request_log_artifact():1525] logged artifact run-p7zqzp4h-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTI3OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTY5NDMxOA==', 'latestArtifact': None}} +2025-01-03 23:09:50,924 DEBUG SenderThread:1992255 [sender.py:send():379] send: files +2025-01-03 23:09:50,924 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json with policy now +2025-01-03 23:09:50,925 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:50,925 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:50,925 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,926 DEBUG SenderThread:1992255 [sender.py:send():379] send: exit +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():586] handling exit code: 0 +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():588] handling runtime: 296 +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:50,926 INFO SenderThread:1992255 [sender.py:send_exit():594] send defer +2025-01-03 23:09:50,926 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,926 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-03 23:09:50,927 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:50,928 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-03 23:09:50,928 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 1 +2025-01-03 23:09:50,928 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,928 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-03 23:09:50,929 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:50,929 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-03 23:09:50,929 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 2 +2025-01-03 23:09:50,929 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [system_monitor.py:finish():203] Stopping system monitor +2025-01-03 23:09:50,929 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-03 23:09:50,929 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined cpu monitor +2025-01-03 23:09:50,929 DEBUG SystemMonitor:1992255 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-03 23:09:50,930 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined disk monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined gpu monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined memory monitor +2025-01-03 23:09:51,007 INFO HandlerThread:1992255 [interfaces.py:finish():200] Joined network monitor +2025-01-03 23:09:51,007 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:51,007 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-03 23:09:51,007 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 3 +2025-01-03 23:09:51,008 DEBUG SenderThread:1992255 [sender.py:send():379] send: stats +2025-01-03 23:09:51,008 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:51,008 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-03 23:09:51,008 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:51,008 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-03 23:09:51,008 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 4 +2025-01-03 23:09:51,008 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:51,008 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-03 23:09:51,156 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/fgooldr4-media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:09:51,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:52,289 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:57,050 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:57,051 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:57,051 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: keepalive +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,052 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-03 23:09:57,052 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 5 +2025-01-03 23:09:57,052 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:57,052 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-03 23:09:57,052 DEBUG SenderThread:1992255 [sender.py:send():379] send: history +2025-01-03 23:09:57,053 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: summary_record +2025-01-03 23:09:57,053 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:57,054 DEBUG SenderThread:1992255 [sender.py:send():379] send: summary +2025-01-03 23:09:57,057 INFO SenderThread:1992255 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-03 23:09:57,058 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 6 +2025-01-03 23:09:57,058 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,058 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-03 23:09:57,058 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,058 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-03 23:09:57,061 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: status_report +2025-01-03 23:09:57,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:57,188 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 7 +2025-01-03 23:09:57,188 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:57,188 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:57,189 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-03 23:09:57,189 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:57,189 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-03 23:09:57,291 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:09:57,292 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:09:58,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:58,631 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 8 +2025-01-03 23:09:58,631 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:09:58,632 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:58,632 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-03 23:09:58,632 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:58,632 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-03 23:09:58,632 INFO SenderThread:1992255 [job_builder.py:build():440] Attempting to build job artifact +2025-01-03 23:09:58,634 INFO SenderThread:1992255 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-03 23:09:58,636 INFO SenderThread:1992255 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-03 23:09:58,655 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 9 +2025-01-03 23:09:58,655 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:09:58,655 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-03 23:09:58,655 DEBUG SenderThread:1992255 [sender.py:send():379] send: artifact +2025-01-03 23:09:59,127 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:09:59,141 INFO wandb-upload_1:1992255 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp87z3f_qi +2025-01-03 23:09:59,141 INFO wandb-upload_0:1992255 [upload_job.py:push():85] Skipped uploading /tmp/tmpzf6e1rpy/wandb-job.json +2025-01-03 23:09:59,238 INFO wandb-upload_2:1992255 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphrizh1qg +2025-01-03 23:09:59,292 INFO Thread-12 :1992255 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxODAxNTQzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxODAwMzc2NA==', 'versionIndex': 414}}} +2025-01-03 23:09:59,873 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-03 23:09:59,873 INFO SenderThread:1992255 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml config.yaml +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json wandb-summary.json +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:10:00,293 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt requirements.txt +2025-01-03 23:10:00,296 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-metadata.json wandb-metadata.json +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log output.log +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/diff.patch diff.patch +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 media/videos/eval/1_eval_1_7cd20e3575921aea286e.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 media/videos/eval/3_eval_3_068501d0d982850a2ba6.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 media/videos/eval/4_eval_4_03a407fa118ffc8cb3c1.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 media/videos/eval/0_eval_0_e28e7a4dc9b13a8d9906.mp4 +2025-01-03 23:10:00,297 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 media/videos/eval/2_eval_2_06a1064193e8e78ad79d.mp4 +2025-01-03 23:10:00,300 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json media/table/eval/success_rate_ci_5_f6f07faa4a586efd9baa.table.json +2025-01-03 23:10:00,302 INFO SenderThread:1992255 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json media/table/eval/total_success_rate_ci_6_412f47f3cc747f52add4.table.json +2025-01-03 23:10:00,302 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 10 +2025-01-03 23:10:00,302 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:00,302 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,302 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-03 23:10:00,303 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,303 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-03 23:10:00,303 INFO SenderThread:1992255 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:10:00,481 INFO wandb-upload_0:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/wandb-summary.json +2025-01-03 23:10:00,502 INFO wandb-upload_1:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/config.yaml +2025-01-03 23:10:00,531 INFO wandb-upload_4:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/output.log +2025-01-03 23:10:00,532 INFO wandb-upload_2:1992255 [upload_job.py:push():130] Uploaded file /tmp/tmpynzrujoywandb/aycs9ny9-events.out.tfevents.1735963493.leonmkim-ROG-Strix-G15CS-G15CS.1992075.0 +2025-01-03 23:10:00,562 INFO wandb-upload_3:1992255 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/files/requirements.txt +2025-01-03 23:10:00,763 INFO Thread-11 (_thread_body):1992255 [sender.py:transition_state():614] send defer: 11 +2025-01-03 23:10:00,763 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,763 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-03 23:10:00,763 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,763 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-03 23:10:00,763 INFO SenderThread:1992255 [file_pusher.py:join():175] waiting for file pusher +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 12 +2025-01-03 23:10:00,765 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,765 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-03 23:10:00,765 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-03 23:10:00,765 INFO SenderThread:1992255 [file_stream.py:finish():601] file stream finish called +2025-01-03 23:10:00,951 INFO SenderThread:1992255 [file_stream.py:finish():605] file stream finish is done +2025-01-03 23:10:00,951 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 13 +2025-01-03 23:10:00,951 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,951 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:transition_state():614] send defer: 14 +2025-01-03 23:10:00,952 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: defer +2025-01-03 23:10:00,952 INFO HandlerThread:1992255 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send():379] send: final +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send():379] send: footer +2025-01-03 23:10:00,952 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: defer +2025-01-03 23:10:00,952 INFO SenderThread:1992255 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-03 23:10:00,953 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:10:00,953 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:00,953 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: server_info +2025-01-03 23:10:00,954 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-03 23:10:00,954 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: server_info +2025-01-03 23:10:00,954 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: get_summary +2025-01-03 23:10:00,956 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-03 23:10:00,956 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-03 23:10:01,010 DEBUG SenderThread:1992255 [sender.py:send_request():406] send_request: poll_exit +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-03 23:10:01,011 INFO MainThread:1992255 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-03 23:10:01,011 DEBUG HandlerThread:1992255 [handler.py:handle_request():158] handle_request: shutdown +2025-01-03 23:10:01,012 INFO HandlerThread:1992255 [handler.py:finish():882] shutting down handler +2025-01-03 23:10:01,954 INFO WriterThread:1992255 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [sender.py:finish():1615] shutting down sender +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [file_pusher.py:finish():169] shutting down file pusher +2025-01-03 23:10:02,011 INFO SenderThread:1992255 [file_pusher.py:join():175] waiting for file pusher diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug.log b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..56693f17fdcbdc6cff5cfc6610279f821ee1c06d --- /dev/null +++ b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Configure stats pid to 1992075 +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/settings +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-03 23:04:52,904 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug.log +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/logs/debug-internal.log +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():569] calling init triggers +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1930_0', 'wandb_notes': '1930_0_restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h'} +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():619] starting backend +2025-01-03 23:04:52,905 INFO MainThread:1992075 [wandb_init.py:init():623] setting up manager +2025-01-03 23:04:52,910 INFO MainThread:1992075 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-03 23:04:52,911 INFO MainThread:1992075 [wandb_init.py:init():631] backend started and connected +2025-01-03 23:04:52,923 INFO MainThread:1992075 [wandb_init.py:init():720] updated telemetry +2025-01-03 23:04:52,929 INFO MainThread:1992075 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-03 23:04:53,209 INFO MainThread:1992075 [wandb_run.py:_on_init():2435] communicating current version +2025-01-03 23:04:53,247 INFO MainThread:1992075 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-03 23:04:53,247 INFO MainThread:1992075 [wandb_init.py:init():804] starting run threads in backend +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_console_start():2413] atexit reg +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-03 23:04:53,641 INFO MainThread:1992075 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-03 23:04:53,642 INFO MainThread:1992075 [wandb_init.py:init():847] run started, returning control to user process +2025-01-03 23:04:53,642 INFO MainThread:1992075 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1930_0/p7zqzp4h/tb, True +2025-01-03 23:05:03,687 INFO MainThread:1992075 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-03 23:10:02,012 WARNING MsgRouterThr:1992075 [router.py:message_loop():77] message_loop has been closed diff --git a/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6ca570187ba2bd0ef7753875c5916cfcc62e5ae4 Binary files /dev/null and b/p7zqzp4h/wandb/run-20250103_230452-p7zqzp4h/run-p7zqzp4h.wandb differ diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4978f5a1c62e29700e82d745dcab76f59310dfe1 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40b1f5f160b55323ccd4f1efc0c9ec77e610985ff37399a9c15fd0ff09a8042f +size 910445958 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b34901953787ce2da565b3dacf86b94a93a55882 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2069f66ff72057d9125005136bf889119896e53433e2527c2e85982e9597f03f +size 910445958 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae266eef9758d512537a7d0adcde81150298de64 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e34b57417272d59f8c39565eddee3d613e9c05afe2b1370ea0543dde48b2e6b8 +size 910445958 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..e08a827233ecea83166c03a96143507c8a004ad2 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27098c6dd5586af176f1b00293fdceff62abcd04e20fd10e118094b46ac5ac1e +size 910445448 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..9678251380563da1a1197f762b76b08c0a6624ed --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e9f3cd587f139c465f5cedfc1295dc4ed3ed78bc9e74fe7df9b0224e7f26ed +size 910445958 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd3faefb63b54dfcd35fd3f288bd1d09183eb334 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ea495178bdc48eae55d873074872b071e0f5efbad5e81e95f4a25a0cb09c07 +size 910445448 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..354ce39a3102d809cd70ee85cf597a831528cc37 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8649148748e6f254c2c10f6523e01e2e1dd0cb5a62c9ff2415b89ba5a6d36396 +size 910445448 diff --git a/snapshot_4737.pt b/snapshot_4737.pt new file mode 100644 index 0000000000000000000000000000000000000000..354ce39a3102d809cd70ee85cf597a831528cc37 --- /dev/null +++ b/snapshot_4737.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8649148748e6f254c2c10f6523e01e2e1dd0cb5a62c9ff2415b89ba5a6d36396 +size 910445448 diff --git a/snapshot_4999.pt b/snapshot_4999.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bb44af18f734bf22e29ca9f752aa7ec713347f7 --- /dev/null +++ b/snapshot_4999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54a33b49b13f58391c6c8c8d1f9264737ef3ad6124a88ba8b581a34d26f0839 +size 910445958 diff --git a/snapshot_5249.pt b/snapshot_5249.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d99828ce008bc217974cc81143752e63e030fd2 --- /dev/null +++ b/snapshot_5249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f4fd01488438c376be909f18b45e265a5ccac72c7926c900758e04ccf292b5 +size 910445958 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e028ff27d16164021871c019989115c70548d261 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17182b7980de971b0b0bca740fe61e97e436f276012333bef2fe852a73a9ed2 +size 910445958 diff --git a/snapshot_6249.pt b/snapshot_6249.pt new file mode 100644 index 0000000000000000000000000000000000000000..0cf8a4015ff01e44473bc1f60b6f36cd851d27c2 --- /dev/null +++ b/snapshot_6249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1990cfd1316b272a9e1ad5ab29b7d8855ffeb1309ed24b6522f83d7cc84d36f5 +size 910445958 diff --git a/snapshot_6999.pt b/snapshot_6999.pt new file mode 100644 index 0000000000000000000000000000000000000000..13a37b27cbda686974616bb672003b16fe6d61dc --- /dev/null +++ b/snapshot_6999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1a3f1417694357e44b0c9acf72b4c70cc4b221cebd2dcabb77b94783f295d1 +size 910445958 diff --git a/snapshot_7499.pt b/snapshot_7499.pt new file mode 100644 index 0000000000000000000000000000000000000000..948cfd5fa9b0749fa6dc1942fa3ce7fc32ad2947 --- /dev/null +++ b/snapshot_7499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe758468443a2b454e0819790e87c305a791a7c845690cf1b6579a075f7cef05 +size 910445958 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dbb79910af300530781fddb04881f88194f4f84 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ee1cd49cd9bdfe086aa66b8ccba752570c58497af87fccf9e1015831e3b6a4 +size 910445958 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ab542e324f19c36da009dac03a59121c1c32f94 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5b96f2322bf280412921d3046f0239a48eeb562e663337440f4eb934b0bfc8 +size 910445958 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..9678251380563da1a1197f762b76b08c0a6624ed --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e9f3cd587f139c465f5cedfc1295dc4ed3ed78bc9e74fe7df9b0224e7f26ed +size 910445958