diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c14d693458baf99a29b1df5e97f64c18495b5a70 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,64 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_7_2025-01-17-17-09-50.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_11_2025-01-17-17-12-47.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_6_2025-01-17-17-09-10.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_16_2025-01-17-17-16-39.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_17_2025-01-17-17-17-33.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_1_2025-01-17-17-05-41.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_18_2025-01-17-17-18-24.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_10_2025-01-17-17-12-00.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_9_2025-01-17-17-11-16.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_8_2025-01-17-17-10-33.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_2_2025-01-17-17-06-25.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_15_2025-01-17-17-15-56.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_3_2025-01-17-17-07-06.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_14_2025-01-17-17-15-07.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_13_2025-01-17-17-14-19.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_19_2025-01-17-17-19-14.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_4_2025-01-17-17-07-48.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_12_2025-01-17-17-13-32.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_5_2025-01-17-17-08-29.bag filter=lfs diff=lfs merge=lfs -text +bgznqd9b/episode_rosbags/episode_0_2025-01-17-17-04-59.bag filter=lfs diff=lfs merge=lfs -text diff --git a/bgznqd9b/.hydra/config.yaml b/bgznqd9b/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fd3a0859b246c9db42fb2b4e2f15f3160aa49a5 --- /dev/null +++ b/bgznqd9b/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bgznqd9b/.hydra/hydra.yaml b/bgznqd9b/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dc797c8c5d71dd0704388c965a683dc239e5bad --- /dev/null +++ b/bgznqd9b/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bgznqd9b/.hydra/overrides.yaml b/bgznqd9b/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bgznqd9b/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bgznqd9b/episode_rosbags/aligned_depth_to_color_K.npy b/bgznqd9b/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bgznqd9b/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bgznqd9b/episode_rosbags/cam_tf_world.npy b/bgznqd9b/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/bgznqd9b/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/bgznqd9b/episode_rosbags/color_K.npy b/bgznqd9b/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bgznqd9b/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bgznqd9b/episode_rosbags/depth_K.npy b/bgznqd9b/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/bgznqd9b/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/bgznqd9b/episode_rosbags/episode_0_2025-01-17-17-04-59.bag b/bgznqd9b/episode_rosbags/episode_0_2025-01-17-17-04-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..70a7e571cae002edfa01acc01b36cb9cfac1e77a --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_0_2025-01-17-17-04-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85d9b8bcc7542088bfdc06331039ed4c42b44a92c62fdec445a4bb77291f7429 +size 1411341289 diff --git a/bgznqd9b/episode_rosbags/episode_10_2025-01-17-17-12-00.bag b/bgznqd9b/episode_rosbags/episode_10_2025-01-17-17-12-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..71b3fa5b7c0cc349aa8d367a830d2bff5671de01 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_10_2025-01-17-17-12-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fef5e2717db0da3e0ebcb1d9a20441d814d293fa48daf072b25d81a080d87b9 +size 1404035876 diff --git a/bgznqd9b/episode_rosbags/episode_11_2025-01-17-17-12-47.bag b/bgznqd9b/episode_rosbags/episode_11_2025-01-17-17-12-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..d8e764caa6e41ef7c1fd2e452d63591f420c1edd --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_11_2025-01-17-17-12-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22b9d80af59f65fb7823d00d100f3ae119ddc305b46e8e593d45269f3d25b0e +size 1409629876 diff --git a/bgznqd9b/episode_rosbags/episode_12_2025-01-17-17-13-32.bag b/bgznqd9b/episode_rosbags/episode_12_2025-01-17-17-13-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..de4abeb58c8f693dc496b6dcb92a8f3c35be6daf --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_12_2025-01-17-17-13-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb008cbb371531bd068bd3e9f5aa66f3041643a9b28d26fbed71aea6a1e08c4 +size 1411956292 diff --git a/bgznqd9b/episode_rosbags/episode_13_2025-01-17-17-14-19.bag b/bgznqd9b/episode_rosbags/episode_13_2025-01-17-17-14-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..adf1a78dd4d8cfadb49ba6d65d9931712f262763 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_13_2025-01-17-17-14-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb46ab9d454a96c395fdb1b38e4d59b37243f26f6c1dac6b7aafa61684817b70 +size 1415816251 diff --git a/bgznqd9b/episode_rosbags/episode_14_2025-01-17-17-15-07.bag b/bgznqd9b/episode_rosbags/episode_14_2025-01-17-17-15-07.bag new file mode 100644 index 0000000000000000000000000000000000000000..2c915fc105385703cc03074ab4c36723b42fad09 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_14_2025-01-17-17-15-07.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31d732681e4e6b55ca3a581f5d30c772f2fc8b57723455e7facbe2975fd63f26 +size 1411300765 diff --git a/bgznqd9b/episode_rosbags/episode_15_2025-01-17-17-15-56.bag b/bgznqd9b/episode_rosbags/episode_15_2025-01-17-17-15-56.bag new file mode 100644 index 0000000000000000000000000000000000000000..3adcf6237cbf94c3b12eec3f63591919fc058024 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_15_2025-01-17-17-15-56.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a70b4e4df77f917269be3b1509a5efe9cd1b15189f4bbaef90b06b5d68cb01 +size 1407860409 diff --git a/bgznqd9b/episode_rosbags/episode_16_2025-01-17-17-16-39.bag b/bgznqd9b/episode_rosbags/episode_16_2025-01-17-17-16-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..0d6628a3a5b9af91cfe771b0f82928794e61cf3a --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_16_2025-01-17-17-16-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517214d387988b45be08f045107d72a0cbb831e344bc17c4dc39456f2e6f8908 +size 1416338237 diff --git a/bgznqd9b/episode_rosbags/episode_17_2025-01-17-17-17-33.bag b/bgznqd9b/episode_rosbags/episode_17_2025-01-17-17-17-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..8a1e030af0fdbe6ce39f1458e9ffce2d77789326 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_17_2025-01-17-17-17-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99e0074c09aacb8cf2a59a55b46fb48b82c37aefcab9fe90ee3a44c2f8a7412b +size 1410710302 diff --git a/bgznqd9b/episode_rosbags/episode_18_2025-01-17-17-18-24.bag b/bgznqd9b/episode_rosbags/episode_18_2025-01-17-17-18-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..449c2ff9b2ab8add99641b613d858e01e7a06f39 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_18_2025-01-17-17-18-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40f0d42b993824b65301c7829c28f69ed0175adff443ee1ad0d79da05490416e +size 1413108692 diff --git a/bgznqd9b/episode_rosbags/episode_19_2025-01-17-17-19-14.bag b/bgznqd9b/episode_rosbags/episode_19_2025-01-17-17-19-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..d9e5374d066efea9025dd2bece5f8400e679fdbd --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_19_2025-01-17-17-19-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86104025cdb35d82eb7e3227cc96e8b621abeeafdb0d7a54b671cdcead884cc1 +size 1407128353 diff --git a/bgznqd9b/episode_rosbags/episode_1_2025-01-17-17-05-41.bag b/bgznqd9b/episode_rosbags/episode_1_2025-01-17-17-05-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..44acb1269c248dec6c9c898335d28dd8926ee334 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_1_2025-01-17-17-05-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a15a7afdf49150163aea039f7d2f22e7b0e6d02005f13aaa9a8f000f32597c86 +size 1406865828 diff --git a/bgznqd9b/episode_rosbags/episode_2_2025-01-17-17-06-25.bag b/bgznqd9b/episode_rosbags/episode_2_2025-01-17-17-06-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..34b04a8813ff74635cc671b0b61bf3e3cc5a0f36 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_2_2025-01-17-17-06-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c21647a93383eda0f4425208c82dbebb3ea25cd3007e235136636252d69f9264 +size 1412203250 diff --git a/bgznqd9b/episode_rosbags/episode_3_2025-01-17-17-07-06.bag b/bgznqd9b/episode_rosbags/episode_3_2025-01-17-17-07-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..955b7e6b6cb52da712c369c83f0fa05acc84cade --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_3_2025-01-17-17-07-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a13f0de8b3e3120024828dfb238f7a1d0ea222d184241c423fce4ca61af1f8cb +size 1414530383 diff --git a/bgznqd9b/episode_rosbags/episode_4_2025-01-17-17-07-48.bag b/bgznqd9b/episode_rosbags/episode_4_2025-01-17-17-07-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..7d99246d4aa6a110f35d2d99d18bd29a4f179322 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_4_2025-01-17-17-07-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:081e9dd89abc71af441c0e5284fb3537cf96324788352ce835870d70c6fa68ab +size 1408151183 diff --git a/bgznqd9b/episode_rosbags/episode_5_2025-01-17-17-08-29.bag b/bgznqd9b/episode_rosbags/episode_5_2025-01-17-17-08-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..27ff51a755cb8b8533461c5a7e0da55946941751 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_5_2025-01-17-17-08-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21da59b54d72e2c5a1168e6e4f45b09b2142cefd8d79947bde4b5515c192c20c +size 1414422187 diff --git a/bgznqd9b/episode_rosbags/episode_6_2025-01-17-17-09-10.bag b/bgznqd9b/episode_rosbags/episode_6_2025-01-17-17-09-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..0d9386c9ebea83ce88a087cf3c00580600b4025c --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_6_2025-01-17-17-09-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85fc65a3e72f6cc05c573dfca10c0d124f05c7669d408a13e6203e6b09f90182 +size 1411492989 diff --git a/bgznqd9b/episode_rosbags/episode_7_2025-01-17-17-09-50.bag b/bgznqd9b/episode_rosbags/episode_7_2025-01-17-17-09-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..590656a2daf621f7797ad0fb37436f56b49c8643 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_7_2025-01-17-17-09-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc339a6f3dda98c76885e35cba360a2fa975f271cf9ba9f76c8260d6238aeba8 +size 1422162858 diff --git a/bgznqd9b/episode_rosbags/episode_8_2025-01-17-17-10-33.bag b/bgznqd9b/episode_rosbags/episode_8_2025-01-17-17-10-33.bag new file mode 100644 index 0000000000000000000000000000000000000000..da76ee35a2acbb220c55c9b9eca21b0bf5a32e22 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_8_2025-01-17-17-10-33.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a8c047d7b5c9aaf145ffe2f418c9e1a3cf07f32905c5479b7f998a37e7caffd +size 1414006039 diff --git a/bgznqd9b/episode_rosbags/episode_9_2025-01-17-17-11-16.bag b/bgznqd9b/episode_rosbags/episode_9_2025-01-17-17-11-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..9ff31ce729e1fbd7205fbc378aa2872e5f4238d4 --- /dev/null +++ b/bgznqd9b/episode_rosbags/episode_9_2025-01-17-17-11-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1985b52982dac9342233e3fee8680121c1db885d6fab1c95381dd532fcf98527 +size 1410991309 diff --git a/bgznqd9b/eval_robot.log b/bgznqd9b/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..753447c69eba4873d58efba24bdfc417b35c55fd --- /dev/null +++ b/bgznqd9b/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-17 17:04:19,284][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-17 17:04:19,287][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/bgznqd9b/eval_video/0_eval.mp4 b/bgznqd9b/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e85abd7850b23dfb0d1987e177041df63e0c229 --- /dev/null +++ b/bgznqd9b/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd6283fb51b44e5730f09910b7a8ad041b98d1c17ba3d18282f631dcd577553c +size 1163211 diff --git a/bgznqd9b/eval_video/10_eval.mp4 b/bgznqd9b/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c00f650affd3c0863e2d6a4d3345e29b079a91d --- /dev/null +++ b/bgznqd9b/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd459ed3c989dccd23a28a6942987a119defe923238ac7c6b9cfdc6b1e49686 +size 1320762 diff --git a/bgznqd9b/eval_video/11_eval.mp4 b/bgznqd9b/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d67ad8d083f047bdea790c13dde22907a267532f --- /dev/null +++ b/bgznqd9b/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa51a8297fd64b89173a1a4fc2e89054789e8805622ff14643ba361e33a1628 +size 1298595 diff --git a/bgznqd9b/eval_video/12_eval.mp4 b/bgznqd9b/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2496cd8dfc722b5fdf6da370f5e1993ca142e340 --- /dev/null +++ b/bgznqd9b/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8267a8b4713015b00d722a44071014f3b9fd05928deb11e665618d20dffc412 +size 1324457 diff --git a/bgznqd9b/eval_video/13_eval.mp4 b/bgznqd9b/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9dc454283607c2b800cd89930f948cbd335a1cfc --- /dev/null +++ b/bgznqd9b/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a3a26bb1dec2c3b14a86672bdd5cc46e2cb3a1b34d08321b5a7b3703576b77 +size 1422027 diff --git a/bgznqd9b/eval_video/14_eval.mp4 b/bgznqd9b/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a0055f1a81b7828c8bae8b8307b0706b4448f66 --- /dev/null +++ b/bgznqd9b/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6742f7ac7cbb27a3fbe88ea98d5d6a88cdebf9150c17531bf94339a98d8d9b65 +size 1149412 diff --git a/bgznqd9b/eval_video/15_eval.mp4 b/bgznqd9b/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..868c2ced6d67f4e4f87571180f44ab621466fc00 --- /dev/null +++ b/bgznqd9b/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deab99abf59c9ff5fce7ea6f8ac1fc88accc09aa6ee77d57e4ef5642dae61546 +size 1154646 diff --git a/bgznqd9b/eval_video/16_eval.mp4 b/bgznqd9b/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65852971cb239f2331a4416f71c5226417b1618b --- /dev/null +++ b/bgznqd9b/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f643ba3fec49b0d46e12670fec854486964eee983a58ab841a3a4583381a6d +size 1475709 diff --git a/bgznqd9b/eval_video/17_eval.mp4 b/bgznqd9b/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc41fed765db07391ba5e25fa11f1eebeae3bedc --- /dev/null +++ b/bgznqd9b/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d21e9b6a3b8f35c0680f91ae15cbff62284e6185af13335704e8fcea9b7432f +size 1486012 diff --git a/bgznqd9b/eval_video/18_eval.mp4 b/bgznqd9b/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..490c0c2eb3a244341e01981c94b04c4c3cc8e751 --- /dev/null +++ b/bgznqd9b/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f12dd2579d2c81aa35439a4b9e23d09db25e600ca8e633a6b027f5325358a67 +size 1488556 diff --git a/bgznqd9b/eval_video/19_eval.mp4 b/bgznqd9b/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e959dd3e616dae6077ed59c50c23a6943ded34c6 --- /dev/null +++ b/bgznqd9b/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5613e6972fd6e3202ca6ab0844726059d4922268113ea6a91640150bd24c1c5 +size 1371732 diff --git a/bgznqd9b/eval_video/1_eval.mp4 b/bgznqd9b/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b7585d93e84c4b238148c615e9f92ff0116b265 --- /dev/null +++ b/bgznqd9b/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7525ad080ceebbb86e1e802b50c8d05120865fcc459fefe0cec41b7f2b712be0 +size 1152019 diff --git a/bgznqd9b/eval_video/2_eval.mp4 b/bgznqd9b/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8aec2f00cc40b422e1b8d03fa737df95033d951 --- /dev/null +++ b/bgznqd9b/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab171c7134ecf8bdca9c97d469120f07014e76518c1405598b99dd4e743ad13 +size 1151271 diff --git a/bgznqd9b/eval_video/3_eval.mp4 b/bgznqd9b/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50ea3759e19dde89b4283262c6b774f58bf3adb7 --- /dev/null +++ b/bgznqd9b/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651f1db6e4859b8c80d688c23d1c03f51b72b3306ec68e1d9f519dd3500e0de3 +size 1195974 diff --git a/bgznqd9b/eval_video/4_eval.mp4 b/bgznqd9b/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17001bfbb5f5583dd2185bb5ecd462db1eb5a0b4 --- /dev/null +++ b/bgznqd9b/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db56d42ec91352f013c01d16154a75ddfab3b3ff2d852f4a3cabbce3d3ea0f18 +size 1159257 diff --git a/bgznqd9b/eval_video/5_eval.mp4 b/bgznqd9b/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d2af334cb769de6c7780b28e92da27e9c043e5a --- /dev/null +++ b/bgznqd9b/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecf6a30ef0ba5b1a068e3e087a267b3d25ca7cb22189074538d1886f34ca504 +size 1164714 diff --git a/bgznqd9b/eval_video/6_eval.mp4 b/bgznqd9b/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af18beba2d251118f9223b149c8459cca3af3465 --- /dev/null +++ b/bgznqd9b/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad60ce20c1e905ff49ab5c56686f1628d52a8c4f296d46b335f1ce2e9b5e782 +size 1166068 diff --git a/bgznqd9b/eval_video/7_eval.mp4 b/bgznqd9b/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea1b3ac03d4cd1e19df38e38d93763d244da2baf --- /dev/null +++ b/bgznqd9b/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ea89fb859f97aaeb81fe8b08d7e5d5848c9d964cb1e6da4f0fe8e6c777cce8 +size 1165224 diff --git a/bgznqd9b/eval_video/8_eval.mp4 b/bgznqd9b/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5f1dc0feeb6c131ab5b502d4b82b499c8fc54e6 --- /dev/null +++ b/bgznqd9b/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33454563ddfcbdb93106c226dddc7e389546fc82939cd0604e55309c12978beb +size 1173628 diff --git a/bgznqd9b/eval_video/9_eval.mp4 b/bgznqd9b/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63525ef6ee834422d3ebf55c67f503a9bfe07495 --- /dev/null +++ b/bgznqd9b/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042356fae54e7c9746ead025ad052e0e8ebc6fdd28da3b42200b5c3494f4687d +size 1302687 diff --git a/bgznqd9b/tb/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 b/bgznqd9b/tb/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 new file mode 100644 index 0000000000000000000000000000000000000000..da0dff7fc6cb9cec3d182a8b9095df96ece808b9 --- /dev/null +++ b/bgznqd9b/tb/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855b2bf5514d9cf6c88ddc799976fdd4fe2fed28acee2327fa5537adb5191c23 +size 22784 diff --git a/bgznqd9b/wandb/debug-internal.log b/bgznqd9b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d6dec93dad258661d067678cc495a765e6d0dec2 --- /dev/null +++ b/bgznqd9b/wandb/debug-internal.log @@ -0,0 +1,2102 @@ +2025-01-17 17:04:25,124 INFO StreamThr :2529582 [internal.py:wandb_internal():85] W&B internal server running at pid: 2529582, started at: 2025-01-17 17:04:25.123689 +2025-01-17 17:04:25,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status +2025-01-17 17:04:25,138 INFO WriterThread:2529582 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb +2025-01-17 17:04:25,138 DEBUG SenderThread:2529582 [sender.py:send():379] send: header +2025-01-17 17:04:25,145 DEBUG SenderThread:2529582 [sender.py:send():379] send: run +2025-01-17 17:04:25,361 INFO SenderThread:2529582 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files +2025-01-17 17:04:25,361 INFO SenderThread:2529582 [sender.py:_start_run_threads():1188] run started: bgznqd9b with start time 1737151465.12452 +2025-01-17 17:04:25,378 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 17:04:25,378 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: check_version +2025-01-17 17:04:25,423 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 17:04:25,428 DEBUG HandlerThread:2529582 [system_info.py:__init__():26] System info init +2025-01-17 17:04:25,428 DEBUG HandlerThread:2529582 [system_info.py:__init__():41] System info init done +2025-01-17 17:04:25,428 INFO HandlerThread:2529582 [system_monitor.py:start():194] Starting system monitor +2025-01-17 17:04:25,428 INFO SystemMonitor:2529582 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 17:04:25,429 INFO HandlerThread:2529582 [system_monitor.py:probe():214] Collecting system info +2025-01-17 17:04:25,429 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 17:04:25,430 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started disk monitoring +2025-01-17 17:04:25,430 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 17:04:25,431 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started memory monitoring +2025-01-17 17:04:25,431 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started network monitoring +2025-01-17 17:04:25,442 DEBUG HandlerThread:2529582 [system_info.py:probe():152] Probing system +2025-01-17 17:04:25,443 DEBUG HandlerThread:2529582 [system_info.py:_probe_git():137] Probing git +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_info.py:_probe_git():145] Probing git done +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_info.py:probe():200] Probing system done +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T22:04:25.442500', 'startedAt': '2025-01-17T22:04:25.108236', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3764.1923749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4533.456, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4521.776, 'min': 800.0, 'max': 4700.0}, {'current': 4532.407, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 469.367977142334}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 17:04:25,449 INFO HandlerThread:2529582 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 17:04:25,449 INFO HandlerThread:2529582 [system_monitor.py:probe():227] Publishing system info +2025-01-17 17:04:25,449 DEBUG HandlerThread:2529582 [system_info.py:_save_code():44] Saving code +2025-01-17 17:04:25,454 DEBUG HandlerThread:2529582 [system_info.py:_save_code():67] Saving code done +2025-01-17 17:04:25,454 DEBUG HandlerThread:2529582 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 17:04:25,549 DEBUG HandlerThread:2529582 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 17:04:25,549 INFO HandlerThread:2529582 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 17:04:25,552 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:04:25,552 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 17:04:25,807 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 17:04:25,807 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: python_packages +2025-01-17 17:04:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:25,808 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/rogd285n-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:04:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:25,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:25,810 INFO HandlerThread:2529582 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/tb" + save: true +} +control { + relay_id: "32f20bf7a49a458eb6e18d914f46c8e1" +} +_info { + stream_id: "bgznqd9b" +} + +2025-01-17 17:04:25,810 DEBUG HandlerThread:2529582 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 17:04:25,815 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/76mzhdsg-code/FISH/eval_robot.py +2025-01-17 17:04:25,830 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/udrg9oq8-wandb-metadata.json +2025-01-17 17:04:25,837 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/1f8b40zh-diff.patch +2025-01-17 17:04:25,914 DEBUG SenderThread:2529582 [sender.py:send():379] send: telemetry +2025-01-17 17:04:25,914 DEBUG SenderThread:2529582 [sender.py:send():379] send: tbrecord +2025-01-17 17:04:25,915 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:04:25,915 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 with policy live +2025-01-17 17:04:26,103 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/dw67t2f5-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:04:26,362 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code +2025-01-17 17:04:26,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:27,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:28,362 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:28,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:29,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:30,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:30,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:31,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:32,713 DEBUG SenderThread:2529582 [sender.py:send():379] send: config +2025-01-17 17:04:32,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:33,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:34,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:34,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:35,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:35,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:36,364 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:36,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:37,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:38,365 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:38,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:39,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:40,365 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:40,807 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:40,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:40,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:40,880 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:41,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:42,366 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:42,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:43,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:44,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:45,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:46,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:46,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:47,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:48,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:49,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:50,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:51,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:52,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:52,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:53,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:54,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:55,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:55,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:55,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:56,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:57,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:58,284 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:58,381 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:58,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:59,381 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:04:59,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:00,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:01,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:02,382 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:02,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:03,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:03,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:04,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:05,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:06,384 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:06,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:07,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:08,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:09,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:09,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:10,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:10,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:10,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:11,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:12,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:13,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:14,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:14,965 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:15,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:16,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:17,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:18,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:19,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:20,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:20,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:21,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:22,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:23,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:24,818 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:25,432 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 17:05:25,432 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:05:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:25,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:25,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:25,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:26,818 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:27,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:27,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:05:27,934 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 with policy now +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:05:27,935 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:05:28,166 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/9rtr7aid-media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos +2025-01-17 17:05:28,662 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/572eui1l-media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,829 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:29,830 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:30,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:30,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:31,513 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:31,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:32,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:33,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:34,392 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:34,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:35,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:36,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:36,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:36,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:37,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:05:37,645 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/4a9r2he9-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:05:37,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:38,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:38,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:39,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:40,394 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:40,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:40,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:40,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:41,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:42,331 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:42,395 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:42,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:43,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:44,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:45,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:46,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:47,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:47,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:48,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:49,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:50,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:51,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:52,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:52,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:53,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:54,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:55,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:05:55,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:55,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:55,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:56,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:57,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:57,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:58,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:59,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:00,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:01,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:02,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:03,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:03,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:04,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:05,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:06,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:07,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:08,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:08,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:09,498 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:06:09,498 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 with policy now +2025-01-17 17:06:09,503 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:06:09,504 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:09,504 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:09,505 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:09,730 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/v56lluot-media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:09,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:10,405 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:10,405 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:10,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:10,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:06:10,681 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/jiv00bf1-media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:10,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:10,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:10,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:11,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:11,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:12,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:13,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:14,161 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:14,407 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:14,852 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:15,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:16,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:17,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:18,408 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:18,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:19,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:19,934 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:20,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:21,409 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:06:21,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:22,292 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:22,293 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:22,294 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:22,409 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:22,410 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:22,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:23,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:24,410 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:24,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:25,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:25,433 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:06:25,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:25,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:25,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:26,411 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:26,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:27,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:28,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:29,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:30,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:30,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:31,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:32,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:33,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:34,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:35,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:35,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:36,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:37,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:38,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:39,844 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:40,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:40,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:40,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:40,928 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:41,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:42,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:43,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:44,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:45,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:46,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:46,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:47,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:48,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:49,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:50,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:51,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:51,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:52,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:53,333 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:06:53,334 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 with policy now +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:53,335 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:06:53,557 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/hjfo797v-media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,824 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/399xn1bs-media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:54,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:54,852 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:55,421 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:55,434 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:06:55,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:55,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:55,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:56,855 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:56,875 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:57,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:58,422 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:58,856 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:59,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:00,856 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:01,855 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:02,423 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:02,423 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:02,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:03,051 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:03,051 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:03,052 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:03,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:03,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:03,752 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/cw6iju6i-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:03,859 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:04,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:04,863 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:05,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:06,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:07,463 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:07,864 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:08,426 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:08,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:09,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:10,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:10,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:10,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:11,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:12,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:12,966 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:13,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:14,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:15,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:16,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:17,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:18,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:18,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:19,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:20,430 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:20,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:21,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:22,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:23,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:23,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:24,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:25,434 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:07:25,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:25,811 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:25,867 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:26,867 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:27,868 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:28,868 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:28,963 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:29,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:30,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:31,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:32,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:33,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:34,258 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:34,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:35,492 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:07:35,492 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 with policy now +2025-01-17 17:07:35,498 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:07:35,499 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:35,499 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:35,499 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:35,712 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/2uyrxgc1-media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:35,878 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:36,435 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:36,435 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:36,436 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:36,436 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:07:36,719 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tz1fsr13-media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:36,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:37,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:38,878 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:39,880 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:40,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:40,437 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:40,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:40,814 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:40,883 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:41,913 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:42,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:43,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:44,438 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:44,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:45,191 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:45,438 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:45,883 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:46,104 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:46,104 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:46,105 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:46,439 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:46,439 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:46,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:47,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:48,440 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:48,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:49,887 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:50,207 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:50,440 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:50,887 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:51,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:52,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:53,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:54,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:55,435 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:07:55,435 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:55,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:55,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:55,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:56,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:57,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:58,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:59,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:00,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:00,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:01,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:02,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:03,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:04,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:05,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:06,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:06,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:07,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:08,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:09,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:10,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:10,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:10,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:11,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:11,935 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:12,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:13,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:14,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:15,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:16,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:17,139 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:17,185 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:08:17,185 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:08:17,186 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 with policy now +2025-01-17 17:08:17,186 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:17,186 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:17,187 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:17,403 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/ms63b0z2-media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:08:17,726 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/llcj3vhz-media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:18,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:18,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:19,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:20,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:21,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:22,452 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:22,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:22,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:23,895 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:24,452 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:24,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:25,435 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:08:25,453 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:08:25,733 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/8hgek1ty-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:08:25,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:25,819 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:25,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:26,152 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:26,153 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:26,153 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:26,453 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:26,454 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:26,904 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:27,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:28,297 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:28,454 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:28,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:29,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:30,455 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:30,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:31,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:32,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:33,823 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:33,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:34,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:35,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:36,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:37,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:38,823 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:38,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:39,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:40,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:40,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:40,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:41,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:42,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:43,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:43,918 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:44,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:45,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:46,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:47,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:48,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:49,385 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:49,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:50,463 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:50,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:51,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:52,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:53,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:54,464 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:54,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:55,186 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:55,436 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:08:55,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:55,813 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:55,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:56,465 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:56,910 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:57,497 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:08:57,497 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 with policy now +2025-01-17 17:08:57,499 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:08:57,500 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:57,500 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:57,500 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:57,711 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tmqsvmfw-media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:57,905 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:58,465 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:08:58,749 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/qkpxgndv-media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:58,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:59,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:59,909 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:00,528 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:00,929 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:01,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:02,467 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:02,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:03,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:04,927 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:05,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:05,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:06,468 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:06,926 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:07,208 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:07,208 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:07,208 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:07,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:07,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:07,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:08,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:08,928 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:09,931 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:10,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:10,816 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:10,914 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:10,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:11,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:12,470 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:12,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:13,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:14,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:15,923 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:16,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:16,923 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:17,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:18,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:19,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:20,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:21,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:22,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:22,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:23,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:24,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:25,436 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:09:25,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:25,816 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:25,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:26,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:27,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:27,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:28,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:29,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:30,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:31,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:32,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:33,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:33,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:34,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:35,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:36,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:37,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:38,480 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:38,757 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:09:38,757 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 with policy now +2025-01-17 17:09:38,761 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:09:38,761 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:38,762 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:38,762 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:38,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:38,972 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/h0247r19-media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:09:39,754 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/vkbl2cmu-media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,779 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:39,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:40,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:40,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:40,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:40,938 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:41,945 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:42,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:43,954 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:44,483 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:44,959 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:45,294 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:45,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:46,254 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:46,255 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:46,272 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:46,746 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/53x0vfc7-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:46,949 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:47,953 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:48,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:48,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:49,945 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:50,384 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:50,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:51,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:52,486 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:52,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:53,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:54,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:55,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:09:55,437 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:55,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:55,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:55,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:56,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:57,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:58,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:59,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:00,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:00,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:01,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:02,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:03,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:04,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:05,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:05,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:06,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:07,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:08,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:09,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:10,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:10,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:10,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:10,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:11,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:12,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:13,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:14,494 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:14,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:15,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:16,828 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:16,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:17,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:18,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:18,599 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:10:18,599 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 with policy now +2025-01-17 17:10:18,602 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:10:18,602 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:10:18,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:10:18,603 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:10:18,833 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/l1gtgiy4-media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:18,948 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:19,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:19,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:10:19,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:19,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:10:19,780 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/wmowzvcs-media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:19,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:20,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:20,960 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:21,836 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:21,961 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:22,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:23,965 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:24,498 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:24,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:25,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:10:25,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:25,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:25,974 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:26,499 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:26,977 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:26,980 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:27,972 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:28,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:29,310 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:10:29,310 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:10:29,311 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:10:29,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:10:29,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:10:29,979 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:30,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:30,983 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:31,985 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:32,501 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:32,851 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:32,978 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:33,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:34,502 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:34,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:35,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:36,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:37,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:38,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:38,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:39,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:40,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:40,819 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:40,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:41,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:42,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:43,931 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:43,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:44,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:45,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:46,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:47,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:48,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:49,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:49,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:50,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:51,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:52,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:53,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:54,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:54,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:55,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:10:55,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:55,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:55,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:56,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:57,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:58,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:59,882 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:59,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:00,980 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:01,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:11:01,444 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 with policy now +2025-01-17 17:11:01,447 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:11:01,447 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:01,448 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:01,448 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:01,512 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:01,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:11:01,673 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xwjsro3l-media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,964 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/j2674ryu-media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,972 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:02,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:02,979 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:03,982 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:04,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:05,102 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:05,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:06,514 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:06,995 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:07,996 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:08,987 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:09,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:10,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:10,516 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:10,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:10,827 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:10,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:11,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:12,364 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:12,365 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:12,365 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:12,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:12,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:12,777 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xf278aws-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:12,992 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:14,000 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:14,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:15,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:15,245 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:16,014 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:17,011 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:18,012 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:18,519 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:19,011 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:20,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:20,520 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:20,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:21,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:22,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:22,521 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:23,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:24,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:25,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:25,438 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:11:25,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:25,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:25,905 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:26,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:27,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:28,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:29,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:30,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:31,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:31,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:32,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:33,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:34,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:35,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:36,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:36,844 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:37,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:38,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:39,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:40,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:40,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:40,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:41,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:41,930 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:42,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:43,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:44,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:45,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:46,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:47,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:47,510 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:47,911 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:11:47,911 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 with policy now +2025-01-17 17:11:47,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:11:47,921 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:47,922 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:47,922 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:48,012 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:48,143 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tpy9igkc-media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:48,532 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:11:48,841 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/fwnoxuum-media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:49,020 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:50,026 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:50,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:51,031 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:52,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:52,532 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:52,621 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:53,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:54,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:55,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:55,438 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:11:55,824 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:55,824 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:56,050 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:56,533 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:57,031 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:57,422 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:57,422 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:57,423 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:57,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:57,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:58,015 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:58,061 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:58,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:59,035 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:00,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:00,535 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:01,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:02,044 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:02,536 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:03,057 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:03,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:04,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:05,037 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:06,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:06,537 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:07,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:08,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:09,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:09,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:10,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:10,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:10,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:11,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:12,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:13,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:14,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:14,541 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:15,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:15,453 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:16,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:17,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:18,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:19,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:20,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:20,857 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:21,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:22,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:23,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:24,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:25,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:25,439 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:12:25,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:25,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:25,896 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:26,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:27,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:28,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:29,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:30,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:31,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:31,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:32,049 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:32,547 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:33,071 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:33,339 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:12:33,340 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 with policy now +2025-01-17 17:12:33,346 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:12:33,346 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:12:33,347 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:12:33,347 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:12:33,606 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/43r1mwhm-media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:33,875 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/v9ypo0s8-media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:34,080 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:34,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:35,086 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:35,549 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:36,072 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:36,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:37,092 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:38,078 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:38,550 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:39,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:40,129 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:40,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:40,845 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:41,097 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:41,934 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:42,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:42,551 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:43,091 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:43,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:44,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:44,485 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:12:44,485 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:12:44,485 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:12:44,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:44,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:44,553 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:12:44,812 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/kzfrmn3i-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:45,101 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:46,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:46,553 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:47,120 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:47,666 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:48,098 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:48,554 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:49,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:50,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:51,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:52,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:52,555 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:52,761 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:53,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:54,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:55,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:55,439 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:12:55,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:55,823 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:56,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:57,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:57,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:58,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:59,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:00,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:00,559 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:01,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:02,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:03,106 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:03,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:04,106 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:05,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:06,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:07,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:08,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:08,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:09,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:10,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:10,563 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:10,824 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:10,824 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:11,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:12,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:13,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:13,974 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:14,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:15,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:16,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:17,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:18,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:19,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:19,257 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:20,030 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:13:20,030 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 with policy now +2025-01-17 17:13:20,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:13:20,046 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:13:20,046 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:13:20,046 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:13:20,114 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:20,258 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/74h79pn1-media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:13:20,848 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/yyc5kh6c-media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:21,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:22,128 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:22,568 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:23,117 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:24,123 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:24,568 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:24,797 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:25,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:25,440 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:13:25,840 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:25,840 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:26,146 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:27,122 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:28,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:28,540 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:13:28,541 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:13:28,541 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:13:29,131 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:30,144 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:30,571 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:30,605 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:31,143 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:32,146 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:33,143 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:34,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:34,572 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:35,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:35,903 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:36,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:37,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:38,147 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:38,574 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:39,147 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:40,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:40,826 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:40,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:40,904 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:41,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:42,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:43,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:44,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:44,576 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:45,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:46,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:46,493 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:46,577 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:47,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:48,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:48,578 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:49,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:50,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:51,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:51,885 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:52,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:53,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:54,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:55,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:55,440 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:13:55,826 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:55,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:56,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:56,917 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:57,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:58,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:59,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:00,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:01,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:02,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:02,885 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:03,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:04,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:05,174 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:05,712 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:14:05,712 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 with policy now +2025-01-17 17:14:05,724 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:14:05,725 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:05,725 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:05,725 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:05,976 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/cqf2nnqn-media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:06,160 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:06,586 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:14:06,873 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/aiwicnmj-media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:07,167 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:07,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:08,189 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:09,164 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:10,186 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:10,587 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:10,830 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:10,831 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:11,178 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:12,190 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:13,167 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:13,509 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:14,182 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:14,588 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:15,181 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:15,589 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:14:15,601 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:15,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:15,602 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:16,182 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:16,589 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:16,590 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:17,207 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:18,189 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:18,590 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:19,210 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:19,223 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:20,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:20,591 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:21,202 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:22,205 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:23,209 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:24,210 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:24,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:25,235 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:25,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:14:25,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:25,849 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:26,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:26,593 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:27,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:28,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:29,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:30,019 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:30,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:31,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:32,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:33,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:34,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:35,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:35,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:36,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:37,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:38,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:39,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:40,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:40,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:40,831 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:40,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:41,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:42,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:43,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:44,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:45,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:46,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:46,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:47,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:48,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:49,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:50,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:51,222 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:51,909 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:52,222 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:53,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:54,249 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:54,475 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:14:54,475 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 with policy now +2025-01-17 17:14:54,478 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:14:54,478 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:54,478 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:54,478 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:14:54,699 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/5e087m64-media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:54,968 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/dxwl07yr-media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:55,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:55,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:14:55,854 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:55,855 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:56,250 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:56,606 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:56,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:57,283 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:58,272 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:59,274 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:00,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:00,607 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:01,257 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:02,258 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:02,262 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:02,608 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:03,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:03,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:03,656 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:03,657 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:03,657 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:03,907 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/q8kjf0e0-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:04,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:04,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:04,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:05,285 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:06,285 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:06,610 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:07,274 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:07,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:08,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:08,611 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:09,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:10,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:10,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:10,842 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:11,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:12,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:12,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:13,267 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:14,267 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:15,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:16,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:17,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:18,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:18,916 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:19,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:20,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:21,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:22,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:23,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:24,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:24,523 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:25,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:25,442 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:15:25,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:25,842 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:26,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:26,618 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:27,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:28,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:29,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:29,975 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:30,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:31,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:32,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:33,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:34,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:35,065 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:35,299 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:35,583 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:15:35,583 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 with policy now +2025-01-17 17:15:35,602 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:15:35,602 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:35,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:35,603 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:35,622 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:35,622 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:35,623 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:15:35,852 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/9yeyz679-media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:36,156 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/a0xfvtn6-media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:36,312 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:36,623 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:37,282 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:38,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:39,329 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:40,311 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:40,329 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:40,624 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:40,850 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:40,850 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:41,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:42,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:43,323 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:44,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:44,626 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:45,301 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:45,452 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:46,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:47,310 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:48,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:49,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:50,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:50,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:51,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:52,335 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:52,629 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:52,629 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:52,714 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:52,714 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:52,714 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:53,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:53,630 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:54,344 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:54,630 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:55,332 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:55,442 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:15:55,892 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:55,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:56,052 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:56,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:57,317 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:58,317 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:58,632 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:59,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:00,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:01,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:01,936 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:02,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:03,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:04,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:05,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:06,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:07,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:07,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:08,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:09,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:10,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:10,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:10,848 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:11,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:12,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:12,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:13,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:14,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:15,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:16,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:17,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:17,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:18,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:19,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:20,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:21,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:22,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:22,938 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:23,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:24,323 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:24,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:24,906 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:16:24,906 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 with policy now +2025-01-17 17:16:24,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:16:24,925 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:16:24,925 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:16:24,925 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:16:25,120 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/6hpol3vb-media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:25,328 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:25,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:16:25,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:16:25,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:25,644 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:16:25,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:25,866 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:25,927 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/mwhuv8dw-media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:26,370 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:26,644 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:27,341 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:27,995 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:28,334 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:29,331 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:30,338 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:30,645 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:31,344 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:32,360 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:32,646 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:33,365 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:33,769 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:34,370 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:35,356 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:35,647 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:16:35,768 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:16:35,768 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:16:35,768 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:16:36,360 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:36,648 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:16:37,349 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:37,648 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:38,368 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:39,333 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:39,377 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:39,649 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:40,368 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:40,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:40,894 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:41,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:41,650 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:42,389 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:43,388 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:44,421 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:44,991 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:45,383 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:46,385 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:47,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:48,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:49,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:49,653 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:50,214 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:50,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:51,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:52,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:53,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:54,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:55,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:55,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:16:55,444 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:55,876 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:55,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:56,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:57,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:58,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:59,374 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:00,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:00,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:01,374 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:02,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:03,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:04,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:05,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:05,948 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:06,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:07,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:08,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:09,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:09,662 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:10,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:10,876 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:10,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:11,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:11,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:12,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:13,377 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:14,387 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:15,238 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:17:15,238 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 with policy now +2025-01-17 17:17:15,252 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:17:15,253 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:17:15,253 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:17:15,253 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:17:15,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:15,461 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/k34rrn1p-media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:17:16,312 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xg3mrrkk-media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:16,406 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:16,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:17,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:17,409 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:17,666 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:18,440 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:19,415 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:20,440 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:21,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:21,667 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:22,023 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:22,412 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:23,429 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:23,668 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:24,425 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:25,427 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:25,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:17:25,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:25,889 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:26,445 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:27,429 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:27,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:28,448 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:29,478 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:29,671 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:17:29,829 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:17:29,830 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:17:29,830 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:17:29,938 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/81nzpym7-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:17:30,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:30,671 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:17:31,455 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:31,672 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:32,451 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:32,978 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:33,448 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:34,468 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:35,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:35,673 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:36,470 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:37,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:38,474 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:38,966 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:39,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:40,475 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:40,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:40,892 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:41,473 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:42,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:43,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:43,677 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:44,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:44,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:45,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:46,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:47,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:48,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:49,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:49,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:50,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:51,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:52,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:53,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:54,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:54,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:55,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:17:55,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:55,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:55,889 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:56,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:57,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:58,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:59,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:00,049 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:00,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:01,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:02,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:03,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:04,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:05,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:05,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:06,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:07,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:08,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:09,483 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:10,490 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:10,561 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:18:10,561 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 with policy now +2025-01-17 17:18:10,586 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:18:10,587 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:18:10,587 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:18:10,587 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:18:10,799 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/b9qauszt-media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:10,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:10,891 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:10,985 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:11,088 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/povne3nd-media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:11,495 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:11,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:12,498 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:12,690 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:13,520 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:13,690 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:14,491 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:15,492 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:15,691 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:16,503 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:16,964 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:17,495 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:18,507 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:19,527 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:19,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:19,891 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:18:19,891 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:18:19,891 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:18:20,505 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:20,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:18:20,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:18:21,524 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:21,694 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:22,069 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:22,520 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:23,516 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:23,695 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:24,508 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:25,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:18:25,518 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:25,695 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:25,906 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:25,906 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:26,577 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:27,533 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:28,044 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:28,549 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:29,534 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:30,541 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:31,568 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:32,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:33,080 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:33,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:34,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:35,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:35,700 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:36,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:37,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:38,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:38,975 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:39,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:40,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:40,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:40,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:41,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:42,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:43,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:44,027 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:44,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:45,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:46,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:47,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:48,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:49,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:49,976 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:50,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:51,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:52,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:53,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:54,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:54,976 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:55,445 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:18:55,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:55,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:55,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:56,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:57,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:58,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:59,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:00,314 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:00,571 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:01,560 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:01,616 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:19:01,616 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 with policy now +2025-01-17 17:19:01,657 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:19:01,657 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:01,658 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:01,658 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:01,711 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:01,711 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:01,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:01,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:19:01,847 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/vnkkhudp-media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:02,161 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/bvgjirbq-media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:02,603 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:03,596 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:03,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:04,601 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:04,713 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:05,343 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:05,604 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:06,612 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:07,605 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:07,714 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:08,657 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:09,608 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:09,953 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:09,953 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:09,954 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:10,637 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:10,715 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:19:10,716 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:10,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:10,922 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:10,992 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:11,684 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:11,716 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:12,640 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:13,642 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:13,717 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:14,637 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:15,702 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:15,718 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:16,002 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:16,675 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:17,685 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:18,705 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:19,699 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:20,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:21,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:21,720 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:21,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:22,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:23,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:23,721 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:24,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:25,445 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:19:25,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:25,901 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:25,902 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:26,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:26,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:27,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:28,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:29,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:30,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:31,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:32,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:32,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:33,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:34,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:35,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:36,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:37,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:37,982 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:38,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:39,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:40,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:40,902 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:40,902 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:41,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:42,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:43,025 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:43,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:44,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:45,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:46,694 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:47,737 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:48,057 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:48,677 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:19:48,677 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 with policy now +2025-01-17 17:19:48,683 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:19:48,683 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:48,684 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:48,684 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:48,732 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:48,732 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:48,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:19:48,748 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:48,897 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/zpvjv7e8-media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:49,164 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/yofnmct4-media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:49,719 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:49,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:50,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:50,739 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:51,734 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:51,762 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:52,734 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:53,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:53,735 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:53,743 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:54,744 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:55,446 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:19:55,747 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:55,917 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:55,918 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:56,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:57,736 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:57,786 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:58,774 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:59,070 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:59,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:00,738 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:00,794 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:01,019 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:01,019 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:01,019 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:01,738 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:01,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:01,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:01,796 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 17:20:01,796 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 17:20:02,191 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:20:02,295 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz47qb_pz +2025-01-17 17:20:02,647 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 17:20:02,656 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media +2025-01-17 17:20:02,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval +2025-01-17 17:20:02,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table +2025-01-17 17:20:02,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:02,868 INFO SenderThread:2529582 [sender.py:send_request_log_artifact():1525] logged artifact run-bgznqd9b-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTM5ODExNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTY5NDg1Mw==', 'latestArtifact': None}} +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:20:02,868 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json with policy now +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,869 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,869 DEBUG SenderThread:2529582 [sender.py:send():379] send: telemetry +2025-01-17 17:20:02,869 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,873 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,873 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,874 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,874 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,874 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,874 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,875 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,875 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,876 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,876 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,877 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,877 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,877 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,877 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,878 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,878 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,878 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,878 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,879 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,879 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 17:20:03,080 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/0te42e0p-media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:03,125 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/haat8bgm-media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:03,127 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/c056iigq-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:03,300 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0g30cvrc +2025-01-17 17:20:03,347 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/i1nwfp2t-media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:03,720 INFO SenderThread:2529582 [sender.py:send_request_log_artifact():1525] logged artifact run-bgznqd9b-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTM5ODEyNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTY5NDg1Ng==', 'latestArtifact': None}} +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:20:03,720 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json with policy now +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:03,721 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:03,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:03,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:03,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:03,976 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/zv8pew0g-media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:04,725 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:04,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:05,741 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:20:05,741 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:05,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:06,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:07,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:08,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:09,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:10,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:10,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:10,902 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:20:10,903 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:20:11,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:12,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:13,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:14,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:15,017 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:15,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:16,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:17,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:18,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:19,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:20,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:21,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:21,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:22,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:23,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:24,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:25,446 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:20:25,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:25,903 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:20:25,903 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:20:26,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:26,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:27,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:28,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:29,237 DEBUG SenderThread:2529582 [sender.py:send():379] send: exit +2025-01-17 17:20:29,237 INFO SenderThread:2529582 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_exit():588] handling runtime: 963 +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_exit():594] send defer +2025-01-17 17:20:29,238 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,238 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 17:20:29,238 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 1 +2025-01-17 17:20:29,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 17:20:29,239 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 2 +2025-01-17 17:20:29,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 17:20:29,239 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 17:20:29,240 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 17:20:29,240 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 17:20:29,240 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined network monitor +2025-01-17 17:20:29,293 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,293 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 17:20:29,293 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 3 +2025-01-17 17:20:29,293 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:20:29,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,293 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 17:20:29,294 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,294 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 17:20:29,294 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 4 +2025-01-17 17:20:29,294 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,294 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 17:20:29,754 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:31,755 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,029 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 17:20:36,029 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 5 +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:36,029 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,030 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:36,030 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 17:20:36,031 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:36,040 DEBUG SenderThread:2529582 [sender.py:send():379] send: summary +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:36,041 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 6 +2025-01-17 17:20:36,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,041 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 17:20:36,041 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 7 +2025-01-17 17:20:36,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:36,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,042 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 17:20:36,042 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 17:20:36,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:36,757 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:37,989 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 8 +2025-01-17 17:20:37,989 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:37,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:37,990 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 17:20:37,990 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:37,990 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 17:20:37,990 INFO SenderThread:2529582 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 17:20:37,993 INFO SenderThread:2529582 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 17:20:37,994 INFO SenderThread:2529582 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 17:20:38,010 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 9 +2025-01-17 17:20:38,010 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:38,010 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 17:20:38,011 DEBUG SenderThread:2529582 [sender.py:send():379] send: artifact +2025-01-17 17:20:38,240 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:38,465 INFO wandb-upload_1:2529582 [upload_job.py:push():85] Skipped uploading /tmp/tmpb4lafne_/wandb-job.json +2025-01-17 17:20:38,468 INFO wandb-upload_3:2529582 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp72azu4yv +2025-01-17 17:20:38,583 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp87v6_dh3 +2025-01-17 17:20:38,758 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTQwMTA5Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTM2NTUwOA==', 'versionIndex': 444}}} +2025-01-17 17:20:39,176 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml config.yaml +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json wandb-summary.json +2025-01-17 17:20:39,760 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt requirements.txt +2025-01-17 17:20:39,761 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json wandb-metadata.json +2025-01-17 17:20:39,762 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:39,763 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:20:39,763 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log output.log +2025-01-17 17:20:39,765 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch diff.patch +2025-01-17 17:20:39,766 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 10 +2025-01-17 17:20:39,768 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:39,768 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:39,768 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 17:20:39,769 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:39,769 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 17:20:39,769 INFO SenderThread:2529582 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 17:20:39,953 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt +2025-01-17 17:20:39,955 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:20:39,978 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:39,983 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:40,183 INFO Thread-11 (_thread_body):2529582 [sender.py:transition_state():614] send defer: 11 +2025-01-17 17:20:40,183 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,183 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 17:20:40,183 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,183 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 17:20:40,183 INFO SenderThread:2529582 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 12 +2025-01-17 17:20:40,187 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,187 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 17:20:40,187 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [file_stream.py:finish():601] file stream finish called +2025-01-17 17:20:40,240 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [file_stream.py:finish():605] file stream finish is done +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 13 +2025-01-17 17:20:40,402 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,402 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,402 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 17:20:40,402 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 14 +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,403 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send():379] send: final +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send():379] send: footer +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,403 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 17:20:40,404 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,404 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: server_info +2025-01-17 17:20:40,405 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,405 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 17:20:40,406 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 17:20:40,407 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:40,460 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,460 INFO MainThread:2529582 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 17:20:40,461 INFO MainThread:2529582 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 17:20:40,461 INFO MainThread:2529582 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 17:20:40,461 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 17:20:40,462 INFO HandlerThread:2529582 [handler.py:finish():882] shutting down handler +2025-01-17 17:20:41,406 INFO WriterThread:2529582 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [sender.py:finish():1615] shutting down sender +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [file_pusher.py:join():175] waiting for file pusher diff --git a/bgznqd9b/wandb/debug.log b/bgznqd9b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..95e34b4e8053f2f0fd443cddd24d9353d86c689e --- /dev/null +++ b/bgznqd9b/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Configure stats pid to 2529391 +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/settings +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug.log +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug-internal.log +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:init():569] calling init triggers +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_2', 'wandb_notes': '3456_2_req_3727_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b'} +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():619] starting backend +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():623] setting up manager +2025-01-17 17:04:25,122 INFO MainThread:2529391 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 17:04:25,124 INFO MainThread:2529391 [wandb_init.py:init():631] backend started and connected +2025-01-17 17:04:25,137 INFO MainThread:2529391 [wandb_init.py:init():720] updated telemetry +2025-01-17 17:04:25,143 INFO MainThread:2529391 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 17:04:25,377 INFO MainThread:2529391 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 17:04:25,418 INFO MainThread:2529391 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 17:04:25,419 INFO MainThread:2529391 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 17:04:25,810 INFO MainThread:2529391 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 17:04:25,810 INFO MainThread:2529391 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/tb, True +2025-01-17 17:04:32,713 INFO MainThread:2529391 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 17:20:41,462 WARNING MsgRouterThr:2529391 [router.py:message_loop():77] message_loop has been closed diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fcf73b387cc93ca67b33089235352af199c2e27 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml @@ -0,0 +1,927 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '3456_2' +wandb_notes: + desc: null + value: 3456_2_req_3727_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: null +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737151465 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..85013ee586477ea44a71897df04e853638fe108c --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch @@ -0,0 +1,80 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 0b377ae..ce79f19 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -132,8 +132,12 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++wandb_run_id: '3456_2' # seed 2 ++ + # RGBD+mask+act history +-wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..5e68f52 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,15 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ '3456_2', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..70cbe8025f462322827a0fbc605dac531a96ee2a --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.8120710253091374, 1, 0.8337996154355976, 1, 0.8628082349170814, 1, 0.9418268053840086, 1]]} \ No newline at end of file diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json new file mode 100644 index 0000000000000000000000000000000000000000..39c8a3a03cea7c1333bf482e8054107008ceff06 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[20, 20, 1.0, 0.8818112611007007, 1, 0.8727449475973412, 1, 0.9030043183951224, 0.9998077586224927, 0.91812883309108, 1]]} \ No newline at end of file diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8e85abd7850b23dfb0d1987e177041df63e0c229 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd6283fb51b44e5730f09910b7a8ad041b98d1c17ba3d18282f631dcd577553c +size 1163211 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3c00f650affd3c0863e2d6a4d3345e29b079a91d --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd459ed3c989dccd23a28a6942987a119defe923238ac7c6b9cfdc6b1e49686 +size 1320762 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d67ad8d083f047bdea790c13dde22907a267532f --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa51a8297fd64b89173a1a4fc2e89054789e8805622ff14643ba361e33a1628 +size 1298595 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2496cd8dfc722b5fdf6da370f5e1993ca142e340 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8267a8b4713015b00d722a44071014f3b9fd05928deb11e665618d20dffc412 +size 1324457 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9dc454283607c2b800cd89930f948cbd335a1cfc --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a3a26bb1dec2c3b14a86672bdd5cc46e2cb3a1b34d08321b5a7b3703576b77 +size 1422027 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a0055f1a81b7828c8bae8b8307b0706b4448f66 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6742f7ac7cbb27a3fbe88ea98d5d6a88cdebf9150c17531bf94339a98d8d9b65 +size 1149412 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..868c2ced6d67f4e4f87571180f44ab621466fc00 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deab99abf59c9ff5fce7ea6f8ac1fc88accc09aa6ee77d57e4ef5642dae61546 +size 1154646 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65852971cb239f2331a4416f71c5226417b1618b --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f643ba3fec49b0d46e12670fec854486964eee983a58ab841a3a4583381a6d +size 1475709 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bc41fed765db07391ba5e25fa11f1eebeae3bedc --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d21e9b6a3b8f35c0680f91ae15cbff62284e6185af13335704e8fcea9b7432f +size 1486012 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..490c0c2eb3a244341e01981c94b04c4c3cc8e751 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f12dd2579d2c81aa35439a4b9e23d09db25e600ca8e633a6b027f5325358a67 +size 1488556 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e959dd3e616dae6077ed59c50c23a6943ded34c6 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5613e6972fd6e3202ca6ab0844726059d4922268113ea6a91640150bd24c1c5 +size 1371732 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b7585d93e84c4b238148c615e9f92ff0116b265 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7525ad080ceebbb86e1e802b50c8d05120865fcc459fefe0cec41b7f2b712be0 +size 1152019 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8aec2f00cc40b422e1b8d03fa737df95033d951 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab171c7134ecf8bdca9c97d469120f07014e76518c1405598b99dd4e743ad13 +size 1151271 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..50ea3759e19dde89b4283262c6b774f58bf3adb7 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651f1db6e4859b8c80d688c23d1c03f51b72b3306ec68e1d9f519dd3500e0de3 +size 1195974 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17001bfbb5f5583dd2185bb5ecd462db1eb5a0b4 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db56d42ec91352f013c01d16154a75ddfab3b3ff2d852f4a3cabbce3d3ea0f18 +size 1159257 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8d2af334cb769de6c7780b28e92da27e9c043e5a --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecf6a30ef0ba5b1a068e3e087a267b3d25ca7cb22189074538d1886f34ca504 +size 1164714 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..af18beba2d251118f9223b149c8459cca3af3465 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad60ce20c1e905ff49ab5c56686f1628d52a8c4f296d46b335f1ce2e9b5e782 +size 1166068 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ea1b3ac03d4cd1e19df38e38d93763d244da2baf --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ea89fb859f97aaeb81fe8b08d7e5d5848c9d964cb1e6da4f0fe8e6c777cce8 +size 1165224 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5f1dc0feeb6c131ab5b502d4b82b499c8fc54e6 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33454563ddfcbdb93106c226dddc7e389546fc82939cd0604e55309c12978beb +size 1173628 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..63525ef6ee834422d3ebf55c67f503a9bfe07495 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042356fae54e7c9746ead025ad052e0e8ebc6fdd28da3b42200b5c3494f4687d +size 1302687 diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..58cf445237c9f137682a6c2d68bda31f497032a9 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log @@ -0,0 +1,1056 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737151472.714546]: resetting environment +[INFO] [1737151472.719512]: cleared current plan +[INFO] [1737151472.720695]: moving to home +[INFO] [1737151473.723072]: reached home +[INFO] [1737151473.724567]: reset action history +[INFO] [1737151476.433113]: environment reset +Starting episode 0 +[INFO] [1737151476.435164]: resetting environment +[INFO] [1737151476.438802]: cleared current plan +[INFO] [1737151476.439761]: moving to home +[INFO] [1737151477.441282]: reached home +[INFO] [1737151477.442621]: reset action history +[INFO] [1737151480.153056]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737151496.232775]: resetting environment +[INFO] [1737151496.236509]: cleared current plan +[INFO] [1737151496.237261]: moving to home +[INFO] [1737151497.239852]: reached home +[INFO] [1737151497.241556]: reset action history +[INFO] [1737151499.949183]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_0_2025-01-17-17-04-59.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:46405/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2529698] +process[print_text-2]: started with pid [2529721] +process[rosbag_record-3]: started with pid [2529722] +started bagging! +For topic gripper_width: timestamp difference is -117637759 for nearest: 1737151504817061300 - target: 1737151504934699059 at idx 60 +[INFO] [1737151526.945180]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1737151527.679435]: Stored episode 1. +[INFO] [1737151527.680276]: Saving video... +[INFO] [1737151527.886375]: Video saved! +Starting episode 1 +[INFO] [1737151527.901775]: resetting environment +[INFO] [1737151527.910696]: cleared current plan +[INFO] [1737151527.910900]: moving to home +[INFO] [1737151531.513121]: reached home +[INFO] [1737151531.513618]: reset action history +[INFO] [1737151534.221602]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151537.389362]: resetting environment +[INFO] [1737151537.391642]: cleared current plan +[INFO] [1737151537.392279]: moving to home +[INFO] [1737151538.395580]: reached home +[INFO] [1737151538.396508]: reset action history +[INFO] [1737151541.104140]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_1_2025-01-17-17-05-41.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:43211/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2529829] +process[print_text-5]: started with pid [2529830] +process[rosbag_record-6]: started with pid [2529831] +started bagging! +[INFO] [1737151568.027518]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737151568.818506]: Stored episode 2. +[INFO] [1737151568.818787]: Saving video... +[INFO] [1737151569.472102]: Video saved! +Starting episode 2 +[INFO] [1737151569.505200]: resetting environment +[INFO] [1737151569.511737]: cleared current plan +[INFO] [1737151569.511921]: moving to home +[WARN] [1737151569.189315]: Plan exhausted +[WARN] [1737151569.229322]: Plan exhausted +[WARN] [1737151569.269449]: Plan exhausted +[WARN] [1737151569.310835]: Plan exhausted +[WARN] [1737151569.349300]: Plan exhausted +[WARN] [1737151569.389322]: Plan exhausted +[WARN] [1737151569.429316]: Plan exhausted +[WARN] [1737151569.470251]: Plan exhausted +[WARN] [1737151569.509459]: Plan exhausted +[INFO] [1737151573.119101]: reached home +[INFO] [1737151573.122681]: reset action history +[INFO] [1737151575.892931]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151581.283430]: resetting environment +[INFO] [1737151581.293511]: cleared current plan +[INFO] [1737151581.293759]: moving to home +[INFO] [1737151582.295681]: reached home +[INFO] [1737151582.296155]: reset action history +[INFO] [1737151585.006397]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_2_2025-01-17-17-06-25.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:41605/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2529966] +process[print_text-8]: started with pid [2529967] +process[rosbag_record-9]: started with pid [2529968] +started bagging! +[INFO] [1737151611.944058]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737151612.737018]: Stored episode 3. +[INFO] [1737151612.741155]: Saving video... +[INFO] [1737151613.290587]: Video saved! +Starting episode 3 +[INFO] [1737151613.322117]: resetting environment +[INFO] [1737151613.331473]: cleared current plan +[INFO] [1737151613.332087]: moving to home +[WARN] [1737151613.293603]: Plan exhausted +[INFO] [1737151616.940109]: reached home +[INFO] [1737151616.944059]: reset action history +[INFO] [1737151619.734367]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151622.437793]: resetting environment +[INFO] [1737151622.449538]: cleared current plan +[INFO] [1737151622.449874]: moving to home +[INFO] [1737151623.452774]: reached home +[INFO] [1737151623.455336]: reset action history +[INFO] [1737151626.227474]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_3_2025-01-17-17-07-06.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:35579/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2530121] +process[print_text-11]: started with pid [2530122] +process[rosbag_record-12]: started with pid [2530123] +started bagging! +For topic gripper_width: timestamp difference is -173072853 for nearest: 1737151639217060051 - target: 1737151639390132904 at idx 156 +[INFO] [1737151654.258360]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1737151655.037169]: Stored episode 4. +[INFO] [1737151655.038933]: Saving video... +[INFO] [1737151655.468034]: Video saved! +Starting episode 4 +[INFO] [1737151655.506377]: resetting environment +[INFO] [1737151655.518870]: cleared current plan +[INFO] [1737151655.523967]: moving to home +[INFO] [1737151659.128201]: reached home +[INFO] [1737151659.131447]: reset action history +[INFO] [1737151661.867626]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151665.215436]: resetting environment +[INFO] [1737151665.226756]: cleared current plan +[INFO] [1737151665.230164]: moving to home +[INFO] [1737151666.233300]: reached home +[INFO] [1737151666.238025]: reset action history +[INFO] [1737151668.961132]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_4_2025-01-17-17-07-48.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:36611/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2530230] +process[print_text-14]: started with pid [2530231] +process[rosbag_record-15]: started with pid [2530232] +started bagging! +[INFO] [1737151695.923716]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1737151696.727574]: Stored episode 5. +[INFO] [1737151696.731420]: Saving video... +[INFO] [1737151697.138357]: Video saved! +Starting episode 5 +[INFO] [1737151697.184330]: resetting environment +[INFO] [1737151697.200197]: cleared current plan +[INFO] [1737151697.203212]: moving to home +[INFO] [1737151700.815613]: reached home +[INFO] [1737151700.819051]: reset action history +[INFO] [1737151703.670464]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151705.228058]: resetting environment +[INFO] [1737151705.240036]: cleared current plan +[INFO] [1737151705.242957]: moving to home +[INFO] [1737151706.248511]: reached home +[INFO] [1737151706.257966]: reset action history +[INFO] [1737151708.990842]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_5_2025-01-17-17-08-29.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:35787/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2530366] +process[print_text-17]: started with pid [2530367] +process[rosbag_record-18]: started with pid [2530368] +started bagging! +For topic gripper_width: timestamp difference is 100292716 for nearest: 1737151729383726869 - target: 1737151729283434153 at idx 249 +For topic gripper_width: timestamp difference is 100719850 for nearest: 1737151729583727519 - target: 1737151729483007669 at idx 249 +For topic gripper_width: timestamp difference is 102575851 for nearest: 1737151731983727289 - target: 1737151731881151438 at idx 249 +For topic gripper_width: timestamp difference is 101460156 for nearest: 1737151732183726487 - target: 1737151732082266331 at idx 249 +For topic gripper_width: timestamp difference is 101589712 for nearest: 1737151735383683952 - target: 1737151735282094240 at idx 249 +[INFO] [1737151735.964534]: Storing episode... +[pub_text-16] killing on exit +[rosbag_record-18] killing on exit +[print_text-17] killing on exit +[INFO] [1737151736.765460]: Stored episode 6. +[INFO] [1737151736.778857]: Saving video... +[INFO] [1737151737.426691]: Video saved! +Starting episode 6 +[INFO] [1737151737.509990]: resetting environment +[INFO] [1737151737.524868]: cleared current plan +[INFO] [1737151737.527356]: moving to home +[WARN] [1737151737.293734]: Plan exhausted +[WARN] [1737151737.335454]: Plan exhausted +[WARN] [1737151737.374537]: Plan exhausted +[WARN] [1737151737.414011]: Plan exhausted +[WARN] [1737151737.457180]: Plan exhausted +[WARN] [1737151737.493192]: Plan exhausted +[INFO] [1737151741.136930]: reached home +[INFO] [1737151741.139822]: reset action history +[INFO] [1737151743.932102]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151746.529635]: resetting environment +[INFO] [1737151746.559312]: cleared current plan +[INFO] [1737151746.563059]: moving to home +[INFO] [1737151747.565203]: reached home +[INFO] [1737151747.576816]: reset action history +[INFO] [1737151750.370105]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_6_2025-01-17-17-09-10.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:37683/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2530521] +process[print_text-20]: started with pid [2530522] +process[rosbag_record-21]: started with pid [2530523] +started bagging! +[INFO] [1737151777.443002]: Storing episode... +[pub_text-19] killing on exit +[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[WARN] [1737151778.614327]: Plan exhausted +[WARN] [1737151778.653238]: Plan exhausted +[WARN] [1737151778.693598]: Plan exhausted +[WARN] [1737151778.733198]: Plan exhausted +[WARN] [1737151778.773191]: Plan exhausted +[INFO] [1737151778.266576]: Stored episode 7. +[INFO] [1737151778.269404]: Saving video... +[INFO] [1737151778.701370]: Video saved! +Starting episode 7 +[INFO] [1737151778.767386]: resetting environment +[INFO] [1737151778.777840]: cleared current plan +[INFO] [1737151778.779039]: moving to home +[INFO] [1737151782.389059]: reached home +[INFO] [1737151782.403863]: reset action history +[INFO] [1737151785.293545]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151786.285141]: resetting environment +[INFO] [1737151786.319032]: cleared current plan +[INFO] [1737151786.333148]: moving to home +[INFO] [1737151787.338825]: reached home +[INFO] [1737151787.347725]: reset action history +[INFO] [1737151790.130110]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_7_2025-01-17-17-09-50.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:42509/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2530652] +process[print_text-23]: started with pid [2530653] +process[rosbag_record-24]: started with pid [2530654] +started bagging! +For topic gripper_width: timestamp difference is -103027334 for nearest: 1737151812017084131 - target: 1737151812120111465 at idx 249 +[INFO] [1737151817.180855]: Storing episode... +[rosbag_record-24] killing on exit +[print_text-23] killing on exit +[pub_text-22] killing on exit +[WARN] [1737151818.533503]: Plan exhausted +[WARN] [1737151818.573188]: Plan exhausted +[WARN] [1737151818.616206]: Plan exhausted +[INFO] [1737151818.125891]: Stored episode 8. +[INFO] [1737151818.137530]: Saving video... +[INFO] [1737151818.543731]: Video saved! +Starting episode 8 +[INFO] [1737151818.607825]: resetting environment +[INFO] [1737151818.642236]: cleared current plan +[INFO] [1737151818.647544]: moving to home +[INFO] [1737151822.258791]: reached home +[INFO] [1737151822.269113]: reset action history +[INFO] [1737151825.132588]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151829.263443]: resetting environment +[INFO] [1737151829.313354]: cleared current plan +[INFO] [1737151829.319132]: moving to home +[INFO] [1737151830.327049]: reached home +[INFO] [1737151830.344321]: reset action history +[INFO] [1737151833.208201]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_8_2025-01-17-17-10-33.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:37669/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2530795] +process[print_text-26]: started with pid [2530796] +process[rosbag_record-27]: started with pid [2530797] +started bagging! +[INFO] [1737151860.323695]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1737151861.139619]: Stored episode 9. +[INFO] [1737151861.150637]: Saving video... +[INFO] [1737151861.389234]: Video saved! +Starting episode 9 +[INFO] [1737151861.452058]: resetting environment +[INFO] [1737151861.486259]: cleared current plan +[INFO] [1737151861.489268]: moving to home +[INFO] [1737151865.101533]: reached home +[INFO] [1737151865.107894]: reset action history +[INFO] [1737151868.059522]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151872.130678]: resetting environment +[INFO] [1737151872.161373]: cleared current plan +[INFO] [1737151872.174535]: moving to home +[INFO] [1737151873.197504]: reached home +[INFO] [1737151873.208866]: reset action history +[INFO] [1737151876.058610]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_9_2025-01-17-17-11-16.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:43309/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2530927] +process[print_text-29]: started with pid [2530928] +process[rosbag_record-30]: started with pid [2530929] +started bagging! +[INFO] [1737151906.704143]: Storing episode... +[rosbag_record-30] killing on exit +[print_text-29] killing on exit +[pub_text-28] killing on exit +[INFO] [1737151907.509558]: Stored episode 10. +[INFO] [1737151907.511786]: Saving video... +[INFO] [1737151907.849190]: Video saved! +Starting episode 10 +[INFO] [1737151907.924154]: resetting environment +[INFO] [1737151907.970912]: cleared current plan +[INFO] [1737151907.973025]: moving to home +[INFO] [1737151911.577217]: reached home +[INFO] [1737151911.582892]: reset action history +[INFO] [1737151914.457263]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151916.960544]: resetting environment +[INFO] [1737151916.996538]: cleared current plan +[INFO] [1737151916.998632]: moving to home +[INFO] [1737151918.014660]: reached home +[INFO] [1737151918.023980]: reset action history +[INFO] [1737151920.876468]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_10_2025-01-17-17-12-00.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:44117/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2531061] +process[print_text-32]: started with pid [2531062] +process[rosbag_record-33]: started with pid [2531063] +started bagging! +For topic gripper_width: timestamp difference is -110069042 for nearest: 1737151933217058176 - target: 1737151933327127218 at idx 177 +[INFO] [1737151951.975747]: Storing episode... +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[print_text-32] killing on exit +[INFO] [1737151952.772886]: Stored episode 11. +[INFO] [1737151952.782946]: Saving video... +[INFO] [1737151953.243268]: Video saved! +Starting episode 11 +[INFO] [1737151953.364075]: resetting environment +[INFO] [1737151953.394413]: cleared current plan +[INFO] [1737151953.408727]: moving to home +[WARN] [1737151953.333208]: Plan exhausted +[WARN] [1737151953.375565]: Plan exhausted +[INFO] [1737151957.030823]: reached home +[INFO] [1737151957.051382]: reset action history +[INFO] [1737151960.086200]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737151963.581041]: resetting environment +[INFO] [1737151963.666837]: cleared current plan +[INFO] [1737151963.672833]: moving to home +[INFO] [1737151964.688266]: reached home +[INFO] [1737151964.710353]: reset action history +[INFO] [1737151967.654996]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_11_2025-01-17-17-12-47.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:38887/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2531195] +process[print_text-35]: started with pid [2531196] +process[rosbag_record-36]: started with pid [2531197] +started bagging! +For topic gripper_width: timestamp difference is -102309574 for nearest: 1737151977917081486 - target: 1737151978019391060 at idx 126 +For topic gripper_width: timestamp difference is 106547501 for nearest: 1737151989117008355 - target: 1737151989010460854 at idx 249 +[INFO] [1737151998.477498]: Storing episode... +[print_text-35] killing on exit +[pub_text-34] killing on exit[rosbag_record-36] killing on exit +[INFO] [1737151999.249162]: Stored episode 12. +[INFO] [1737151999.266474]: Saving video... +[INFO] [1737151999.943594]: Video saved! +[WARN] [1737151999.814023]: Plan exhausted +[WARN] [1737151999.854719]: Plan exhausted +[WARN] [1737151999.893954]: Plan exhausted +[WARN] [1737151999.934570]: Plan exhausted +[WARN] [1737151999.975711]: Plan exhausted +[WARN] [1737152000.015518]: Plan exhausted +[WARN] [1737152000.053574]: Plan exhausted +[WARN] [1737152000.093919]: Plan exhausted +Starting episode 12 +[INFO] [1737152000.067360]: resetting environment +[INFO] [1737152000.114740]: cleared current plan +[INFO] [1737152000.132278]: moving to home +[INFO] [1737152003.754519]: reached home +[INFO] [1737152003.766538]: reset action history +[INFO] [1737152006.816358]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152008.518121]: resetting environment +[INFO] [1737152008.536458]: cleared current plan +[INFO] [1737152008.544870]: moving to home +[INFO] [1737152009.553365]: reached home +[INFO] [1737152009.580752]: reset action history +[INFO] [1737152012.526108]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_12_2025-01-17-17-13-32.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:45721/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2531329] +process[print_text-38]: started with pid [2531330] +process[rosbag_record-39]: started with pid [2531331] +started bagging! +For topic gripper_width: timestamp difference is 100068146 for nearest: 1737152022283724123 - target: 1737152022183655977 at idx 144 +For topic gripper_width: timestamp difference is 101907948 for nearest: 1737152024683761338 - target: 1737152024581853390 at idx 169 +For topic gripper_width: timestamp difference is 102034085 for nearest: 1737152024883726351 - target: 1737152024781692266 at idx 171 +For topic gripper_width: timestamp difference is -108408781 for nearest: 1737152026317056088 - target: 1737152026425464869 at idx 191 +For topic gripper_width: timestamp difference is -105554700 for nearest: 1737152027717057586 - target: 1737152027822612286 at idx 207 +[INFO] [1737152044.466992]: Storing episode... +[rosbag_record-39] killing on exit +[pub_text-37] killing on exit +[print_text-38] killing on exit +[INFO] [1737152045.259949]: Stored episode 13. +[INFO] [1737152045.276497]: Saving video... +[INFO] [1737152045.617143]: Video saved! +Starting episode 13 +[INFO] [1737152045.728492]: resetting environment +[INFO] [1737152045.761266]: cleared current plan +[INFO] [1737152045.765571]: moving to home +[INFO] [1737152049.386350]: reached home +[INFO] [1737152049.395409]: reset action history +[INFO] [1737152052.460830]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152055.238648]: resetting environment +[INFO] [1737152055.246208]: cleared current plan +[INFO] [1737152055.246462]: moving to home +[INFO] [1737152056.249034]: reached home +[INFO] [1737152056.284868]: reset action history +[INFO] [1737152059.217347]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_13_2025-01-17-17-14-19.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:36801/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2531464] +process[print_text-41]: started with pid [2531465] +process[rosbag_record-42]: started with pid [2531466] +started bagging! +[INFO] [1737152092.728313]: Storing episode... +[rosbag_record-42] killing on exit +[pub_text-40] killing on exit +[print_text-41] killing on exit +[INFO] [1737152093.633396]: Stored episode 14. +[INFO] [1737152093.650078]: Saving video... +[WARN] [1737152094.094161]: Plan exhausted +[WARN] [1737152094.135236]: Plan exhausted +[WARN] [1737152094.175652]: Plan exhausted +[WARN] [1737152094.215878]: Plan exhausted +[WARN] [1737152094.255304]: Plan exhausted +[WARN] [1737152094.311316]: Plan exhausted +[WARN] [1737152094.353249]: Plan exhausted +[WARN] [1737152094.373411]: Plan exhausted +[WARN] [1737152094.419431]: Plan exhausted +[WARN] [1737152094.453640]: Plan exhausted +[WARN] [1737152094.494154]: Plan exhausted +[INFO] [1737152094.324655]: Video saved! +Starting episode 14 +[INFO] [1737152094.512415]: resetting environment +[INFO] [1737152094.531890]: cleared current plan +[INFO] [1737152094.538823]: moving to home +[INFO] [1737152098.152752]: reached home +[INFO] [1737152098.168051]: reset action history +[INFO] [1737152101.206344]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152103.174941]: resetting environment +[INFO] [1737152103.228596]: cleared current plan +[INFO] [1737152103.249438]: moving to home +[INFO] [1737152104.264915]: reached home +[INFO] [1737152104.283009]: reset action history +[INFO] [1737152107.306255]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_14_2025-01-17-17-15-07.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:46169/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2531573] +process[print_text-44]: started with pid [2531574] +process[rosbag_record-45]: started with pid [2531598] +started bagging! +For topic gripper_width: timestamp difference is 114166446 for nearest: 1737152124517075010 - target: 1737152124402908564 at idx 239 +[INFO] [1737152134.279260]: Storing episode... +[print_text-44] killing on exit +[rosbag_record-45] killing on exit +[pub_text-43] killing on exit +[INFO] [1737152135.056020]: Stored episode 15. +[INFO] [1737152135.077961]: Saving video... +[INFO] [1737152135.488400]: Video saved! +Starting episode 15 +[INFO] [1737152135.609994]: resetting environment +[INFO] [1737152135.629541]: cleared current plan +[INFO] [1737152135.640934]: moving to home +[INFO] [1737152139.277584]: reached home +[INFO] [1737152139.302684]: reset action history +[INFO] [1737152142.395585]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152152.301465]: resetting environment +[INFO] [1737152152.370168]: cleared current plan +[INFO] [1737152152.381631]: moving to home +[INFO] [1737152153.403105]: reached home +[INFO] [1737152153.411033]: reset action history +[INFO] [1737152156.482835]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_15_2025-01-17-17-15-56.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:42425/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2531706] +process[print_text-47]: started with pid [2531707] +process[rosbag_record-48]: started with pid [2531731] +started bagging! +[INFO] [1737152183.476483]: Storing episode... +[rosbag_record-48] killing on exit +[print_text-47] killing on exit +[pub_text-46] killing on exit +[WARN] [1737152184.813657]: Plan exhausted +[WARN] [1737152184.854126]: Plan exhausted +[WARN] [1737152184.895117]: Plan exhausted +[WARN] [1737152184.933829]: Plan exhausted +[WARN] [1737152184.976455]: Plan exhausted +[INFO] [1737152184.312723]: Stored episode 16. +[INFO] [1737152184.316794]: Saving video... +[INFO] [1737152184.779522]: Video saved! +Starting episode 16 +[INFO] [1737152184.963507]: resetting environment +[INFO] [1737152184.993352]: cleared current plan +[INFO] [1737152184.994852]: moving to home +[INFO] [1737152188.624055]: reached home +[INFO] [1737152188.639621]: reset action history +[INFO] [1737152191.719412]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152195.160255]: resetting environment +[INFO] [1737152195.224742]: cleared current plan +[INFO] [1737152195.241509]: moving to home +[INFO] [1737152196.289391]: reached home +[INFO] [1737152196.306139]: reset action history +[INFO] [1737152199.366430]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_16_2025-01-17-17-16-39.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:40069/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2531861] +process[print_text-50]: started with pid [2531862] +process[rosbag_record-51]: started with pid [2531863] +started bagging! +For topic gripper_width: timestamp difference is -135184846 for nearest: 1737152226117055812 - target: 1737152226252240658 at idx 249 +[INFO] [1737152233.925875]: Storing episode... +[rosbag_record-51] killing on exit +[print_text-50] killing on exit +[pub_text-49] killing on exit +[WARN] [1737152235.257140]: Plan exhausted +[WARN] [1737152235.297056]: Plan exhausted +[INFO] [1737152234.808879]: Stored episode 17. +[INFO] [1737152234.823539]: Saving video... +[INFO] [1737152235.119106]: Video saved! +Starting episode 17 +[INFO] [1737152235.278123]: resetting environment +[INFO] [1737152235.327049]: cleared current plan +[INFO] [1737152235.331419]: moving to home +[INFO] [1737152238.972772]: reached home +[INFO] [1737152238.992217]: reset action history +[INFO] [1737152242.018060]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152248.989078]: resetting environment +[INFO] [1737152249.054675]: cleared current plan +[INFO] [1737152249.056599]: moving to home +[INFO] [1737152250.068591]: reached home +[INFO] [1737152250.094247]: reset action history +[INFO] [1737152253.138498]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_17_2025-01-17-17-17-33.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:35231/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2531998] +process[print_text-53]: started with pid [2531999] +process[rosbag_record-54]: started with pid [2532000] +started bagging! +[INFO] [1737152288.993897]: Storing episode... +[print_text-53] killing on exit +[pub_text-52] killing on exit +[rosbag_record-54] killing on exit +[INFO] [1737152289.881534]: Stored episode 18. +[INFO] [1737152289.920501]: Saving video... +[WARN] [1737152290.337564]: Plan exhausted +[WARN] [1737152290.379452]: Plan exhausted +[WARN] [1737152290.413327]: Plan exhausted +[WARN] [1737152290.477149]: Plan exhausted +[WARN] [1737152290.496793]: Plan exhausted +[WARN] [1737152290.533203]: Plan exhausted +[WARN] [1737152290.573198]: Plan exhausted +[WARN] [1737152290.616590]: Plan exhausted +[INFO] [1737152290.455506]: Video saved! +Starting episode 18 +[INFO] [1737152290.596783]: resetting environment +[INFO] [1737152290.634311]: cleared current plan +[INFO] [1737152290.638752]: moving to home +[INFO] [1737152294.255658]: reached home +[INFO] [1737152294.295549]: reset action history +[INFO] [1737152297.590771]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152299.893308]: resetting environment +[INFO] [1737152299.989205]: cleared current plan +[INFO] [1737152300.012101]: moving to home +[INFO] [1737152301.027445]: reached home +[INFO] [1737152301.051216]: reset action history +[INFO] [1737152304.204366]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_18_2025-01-17-17-18-24.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:34781/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2532131] +process[print_text-56]: started with pid [2532132] +process[rosbag_record-57]: started with pid [2532133] +started bagging! +[INFO] [1737152340.313564]: Storing episode... +[rosbag_record-57] killing on exit +[print_text-56] killing on exit +[pub_text-55] killing on exit +[INFO] [1737152341.156575]: Stored episode 19. +[INFO] [1737152341.182768]: Saving video... +[INFO] [1737152341.485009]: Video saved! +Starting episode 19 +[INFO] [1737152341.644142]: resetting environment +[INFO] [1737152341.689815]: cleared current plan +[INFO] [1737152341.691705]: moving to home +[WARN] [1737152341.623748]: Plan exhausted +[WARN] [1737152341.655851]: Plan exhausted +[INFO] [1737152345.309274]: reached home +[INFO] [1737152345.347695]: reset action history +[INFO] [1737152348.615965]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737152349.935313]: resetting environment +[INFO] [1737152350.035049]: cleared current plan +[INFO] [1737152350.037129]: moving to home +[INFO] [1737152351.075768]: reached home +[INFO] [1737152351.091928]: reset action history +[INFO] [1737152354.357288]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/episode_rosbags/episode_19_2025-01-17-17-19-14.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2529391.log +started roslaunch server http://158.130.50.37:43357/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2532263] +process[print_text-59]: started with pid [2532264] +process[rosbag_record-60]: started with pid [2532265] +started bagging! +For topic gripper_width: timestamp difference is 106883526 for nearest: 1737152360783720970 - target: 1737152360676837444 at idx 95 +[INFO] [1737152387.178972]: Storing episode... +[rosbag_record-60] killing on exit +[pub_text-58] killing on exit +[print_text-59] killing on exit +[INFO] [1737152388.011724]: Stored episode 20. +[INFO] [1737152388.037702]: Saving video... +[WARN] [1737152388.543957]: Plan exhausted +[WARN] [1737152388.574563]: Plan exhausted +[WARN] [1737152388.614978]: Plan exhausted +[WARN] [1737152388.657340]: Plan exhausted +[WARN] [1737152388.698559]: Plan exhausted +[WARN] [1737152388.739474]: Plan exhausted +[WARN] [1737152388.773890]: Plan exhausted +[INFO] [1737152388.513032]: Video saved! +[INFO] [1737152388.709895]: resetting environment +[INFO] [1737152388.800746]: cleared current plan +[INFO] [1737152388.806200]: moving to home +[INFO] [1737152392.415541]: reached home +[INFO] [1737152392.440974]: reset action history +[INFO] [1737152395.696803]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch new file mode 100644 index 0000000000000000000000000000000000000000..00e358bb41a9ecd35e13ffb3d779a067da0ab748 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch @@ -0,0 +1,97 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..ce79f19 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -132,15 +132,19 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '1000_0' # all + + # all books, 10/20 demos per book ++# crp_D+act history ++# wandb_run_id: '3456_0' ++# wandb_run_id: '3456_1' # seed 1 ++wandb_run_id: '3456_2' # seed 2 ++ + # RGBD+mask+act history +-# wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index e3c7dc1..5e68f52 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -167,12 +167,15 @@ run_id_list = [ + # '1948_0', + # '1950_0', + # '1954_1', +- '1002_0', +- '1009_1', +- '1022_0', +- '1003_0', +- '1013_1', +- '1017_0', ++ # '1002_0', ++ # '1009_1', ++ # '1022_0', ++ # '1003_0', ++ # '1013_1', ++ # '1017_0', ++ # '3456_0', ++ # '3456_1', ++ '3456_2', + ] + + checkpoint_epoch = 12000 +diff --git a/FISH/eval_policy.py b/FISH/eval_policy.py +index 0ad179a..f5cfc13 100644 +--- a/FISH/eval_policy.py ++++ b/FISH/eval_policy.py +@@ -89,9 +89,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: +diff --git a/FISH/eval_robot.py b/FISH/eval_robot.py +index 50493fe..905b769 100644 +--- a/FISH/eval_robot.py ++++ b/FISH/eval_robot.py +@@ -83,9 +83,10 @@ class Workspace: + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( +- hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), +- self.cfg.agent.config.policy_cfg.input_shapes, +- hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, ++ hydra.utils.instantiate(self.cfg.agent.config), ++ # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), ++ # self.cfg.agent.config.policy_cfg.input_shapes, ++ # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..15033aad0b736743e8daf0704dcdbeecd75bec22 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-17T22:04:25.442500", + "startedAt": "2025-01-17T22:04:25.108236", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "ec42957dc10d08dbd8b134558fcd8f9e470ccc31" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3764.1923749999996, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4533.456, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4521.776, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4532.407, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 469.367977142334 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed344e241baa7704cb564bd4a9363b50c6059d7 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "cd6283fb51b44e5730f09910b7a8ad041b98d1c17ba3d18282f631dcd577553c", "size": 1163211, "path": "media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4"}, "global_step": 2658, "_timestamp": 1737152402.5458896, "_runtime": 937.4213695526123, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "7525ad080ceebbb86e1e802b50c8d05120865fcc459fefe0cec41b7f2b712be0", "size": 1152019, "path": "media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4"}, "eval/num_success": 20.0, "episode": 20.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.8120710253715515, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.8337996006011963, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.8628082275390625, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.9418268203735352, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "8ab171c7134ecf8bdca9c97d469120f07014e76518c1405598b99dd4e743ad13", "size": 1151271, "path": "media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "651f1db6e4859b8c80d688c23d1c03f51b72b3306ec68e1d9f519dd3500e0de3", "size": 1195974, "path": "media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "db56d42ec91352f013c01d16154a75ddfab3b3ff2d852f4a3cabbce3d3ea0f18", "size": 1159257, "path": "media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "7ecf6a30ef0ba5b1a068e3e087a267b3d25ca7cb22189074538d1886f34ca504", "size": 1164714, "path": "media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "9ad60ce20c1e905ff49ab5c56686f1628d52a8c4f296d46b335f1ce2e9b5e782", "size": 1166068, "path": "media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "38ea89fb859f97aaeb81fe8b08d7e5d5848c9d964cb1e6da4f0fe8e6c777cce8", "size": 1165224, "path": "media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "33454563ddfcbdb93106c226dddc7e389546fc82939cd0604e55309c12978beb", "size": 1173628, "path": "media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "042356fae54e7c9746ead025ad052e0e8ebc6fdd28da3b42200b5c3494f4687d", "size": 1302687, "path": "media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "9dd459ed3c989dccd23a28a6942987a119defe923238ac7c6b9cfdc6b1e49686", "size": 1320762, "path": "media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "8aa51a8297fd64b89173a1a4fc2e89054789e8805622ff14643ba361e33a1628", "size": 1298595, "path": "media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "b8267a8b4713015b00d722a44071014f3b9fd05928deb11e665618d20dffc412", "size": 1324457, "path": "media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "85a3a26bb1dec2c3b14a86672bdd5cc46e2cb3a1b34d08321b5a7b3703576b77", "size": 1422027, "path": "media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "6742f7ac7cbb27a3fbe88ea98d5d6a88cdebf9150c17531bf94339a98d8d9b65", "size": 1149412, "path": "media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "deab99abf59c9ff5fce7ea6f8ac1fc88accc09aa6ee77d57e4ef5642dae61546", "size": 1154646, "path": "media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "e9f643ba3fec49b0d46e12670fec854486964eee983a58ab841a3a4583381a6d", "size": 1475709, "path": "media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "0d21e9b6a3b8f35c0680f91ae15cbff62284e6185af13335704e8fcea9b7432f", "size": 1486012, "path": "media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "8f12dd2579d2c81aa35439a4b9e23d09db25e600ca8e633a6b027f5325358a67", "size": 1488556, "path": "media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "c5613e6972fd6e3202ca6ab0844726059d4922268113ea6a91640150bd24c1c5", "size": 1371732, "path": "media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "203f9a9e17ed51251fddaf31c2da0334525164164e50ce4dec7a55d16f5ed83e", "size": 330, "artifact_path": "wandb-client-artifact://ljd14mhataowtmzd29v2iuuy8bhw4t1pd0zntntp1b0z7tzohamdqlwf5ea1sfjxwu0jwwekmuxqcwg63imhzxis5w9p3v7ni1sc9eq8gqjcl0ptvd1zruaguvmmnsi2/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://0azxt6e40fey2mclvrnnprx7rk0uyx1ppmsc8cia1sd0cjko2zp1czc1wdwyih0m6zb08aj1phwetgftsuqoi57po5nqvt9xn64hl9p0yv1iqnuzpd6d5nltsdw3o0ga:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 20, "total_num_episodes": 20, "total_success_rate": 1.0, "total_success_rate_lb_0.01": 0.8818112611007007, "total_success_rate_ub_0.01": 1, "total_success_rate_lb_0.025": 0.8727449475973412, "total_success_rate_ub_0.025": 1, "total_success_rate_lb_0.05": 0.9030043183951224, "total_success_rate_ub_0.05": 0.9998077586224927, "total_success_rate_lb_0.1": 0.91812883309108, "total_success_rate_ub_0.1": 1, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "99066e668ec75c97d0dd36f4085d4f0016f54007b215991eb2d347bf65aea69f", "size": 452, "artifact_path": "wandb-client-artifact://8mmb77hd8w9bgd53ln35pr0f6ysughw62nln6zjoe68awodc17mil1qvgon4vj53ep4vdil1lwnh64h5ndxv1o9ln3m42wu4csy66k2gm046okuwwd9xyidtepp4wwz2/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://y3qga9qhjryz4zxx5j5fnm8f7om4y32m2pnp2sd3ljtpef2ugm9kj5pge42kv66lmvx611p00xiyq8y0fwij6jvv87o6zdin15s6i9qgivdkegwbos79ii5p2ndfbrws:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 963}, "eval/total_success_rate": 1.0, "eval/total_success_rate_lb_0.01": 0.881811261177063, "eval/total_success_rate_ub_0.01": 1.0, "eval/total_success_rate_lb_0.025": 0.8727449178695679, "eval/total_success_rate_ub_0.025": 1.0, "eval/total_success_rate_lb_0.05": 0.9030042886734009, "eval/total_success_rate_ub_0.05": 0.9998077750205994, "eval/total_success_rate_lb_0.1": 0.9181288480758667, "eval/total_success_rate_ub_0.1": 1.0} \ No newline at end of file diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug-internal.log b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d6dec93dad258661d067678cc495a765e6d0dec2 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug-internal.log @@ -0,0 +1,2102 @@ +2025-01-17 17:04:25,124 INFO StreamThr :2529582 [internal.py:wandb_internal():85] W&B internal server running at pid: 2529582, started at: 2025-01-17 17:04:25.123689 +2025-01-17 17:04:25,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status +2025-01-17 17:04:25,138 INFO WriterThread:2529582 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb +2025-01-17 17:04:25,138 DEBUG SenderThread:2529582 [sender.py:send():379] send: header +2025-01-17 17:04:25,145 DEBUG SenderThread:2529582 [sender.py:send():379] send: run +2025-01-17 17:04:25,361 INFO SenderThread:2529582 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files +2025-01-17 17:04:25,361 INFO SenderThread:2529582 [sender.py:_start_run_threads():1188] run started: bgznqd9b with start time 1737151465.12452 +2025-01-17 17:04:25,378 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: check_version +2025-01-17 17:04:25,378 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: check_version +2025-01-17 17:04:25,423 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: run_start +2025-01-17 17:04:25,428 DEBUG HandlerThread:2529582 [system_info.py:__init__():26] System info init +2025-01-17 17:04:25,428 DEBUG HandlerThread:2529582 [system_info.py:__init__():41] System info init done +2025-01-17 17:04:25,428 INFO HandlerThread:2529582 [system_monitor.py:start():194] Starting system monitor +2025-01-17 17:04:25,428 INFO SystemMonitor:2529582 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-17 17:04:25,429 INFO HandlerThread:2529582 [system_monitor.py:probe():214] Collecting system info +2025-01-17 17:04:25,429 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started cpu monitoring +2025-01-17 17:04:25,430 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started disk monitoring +2025-01-17 17:04:25,430 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started gpu monitoring +2025-01-17 17:04:25,431 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started memory monitoring +2025-01-17 17:04:25,431 INFO SystemMonitor:2529582 [interfaces.py:start():188] Started network monitoring +2025-01-17 17:04:25,442 DEBUG HandlerThread:2529582 [system_info.py:probe():152] Probing system +2025-01-17 17:04:25,443 DEBUG HandlerThread:2529582 [system_info.py:_probe_git():137] Probing git +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_info.py:_probe_git():145] Probing git done +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_info.py:probe():200] Probing system done +2025-01-17 17:04:25,448 DEBUG HandlerThread:2529582 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-17T22:04:25.442500', 'startedAt': '2025-01-17T22:04:25.108236', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'ec42957dc10d08dbd8b134558fcd8f9e470ccc31'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3764.1923749999996, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4533.456, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4521.776, 'min': 800.0, 'max': 4700.0}, {'current': 4532.407, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 469.367977142334}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-17 17:04:25,449 INFO HandlerThread:2529582 [system_monitor.py:probe():224] Finished collecting system info +2025-01-17 17:04:25,449 INFO HandlerThread:2529582 [system_monitor.py:probe():227] Publishing system info +2025-01-17 17:04:25,449 DEBUG HandlerThread:2529582 [system_info.py:_save_code():44] Saving code +2025-01-17 17:04:25,454 DEBUG HandlerThread:2529582 [system_info.py:_save_code():67] Saving code done +2025-01-17 17:04:25,454 DEBUG HandlerThread:2529582 [system_info.py:_save_patches():84] Saving git patches +2025-01-17 17:04:25,549 DEBUG HandlerThread:2529582 [system_info.py:_save_patches():126] Saving git patches done +2025-01-17 17:04:25,549 INFO HandlerThread:2529582 [system_monitor.py:probe():229] Finished publishing system info +2025-01-17 17:04:25,552 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:04:25,552 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-17 17:04:25,553 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch with policy now +2025-01-17 17:04:25,807 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: python_packages +2025-01-17 17:04:25,807 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: python_packages +2025-01-17 17:04:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:25,808 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/rogd285n-upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:04:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:25,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:25,810 INFO HandlerThread:2529582 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/tb" + save: true +} +control { + relay_id: "32f20bf7a49a458eb6e18d914f46c8e1" +} +_info { + stream_id: "bgznqd9b" +} + +2025-01-17 17:04:25,810 DEBUG HandlerThread:2529582 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-17 17:04:25,815 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/76mzhdsg-code/FISH/eval_robot.py +2025-01-17 17:04:25,830 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/udrg9oq8-wandb-metadata.json +2025-01-17 17:04:25,837 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/1f8b40zh-diff.patch +2025-01-17 17:04:25,914 DEBUG SenderThread:2529582 [sender.py:send():379] send: telemetry +2025-01-17 17:04:25,914 DEBUG SenderThread:2529582 [sender.py:send():379] send: tbrecord +2025-01-17 17:04:25,915 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:04:25,915 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 with policy live +2025-01-17 17:04:26,103 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/dw67t2f5-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:04:26,362 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH +2025-01-17 17:04:26,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code +2025-01-17 17:04:26,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:27,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:28,362 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:28,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:29,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:30,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:30,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:31,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:32,713 DEBUG SenderThread:2529582 [sender.py:send():379] send: config +2025-01-17 17:04:32,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:33,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:34,363 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:34,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:35,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:35,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:36,364 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:36,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:37,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:38,365 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:38,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:39,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:40,365 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:40,807 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:40,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:40,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:40,880 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:41,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:42,366 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:42,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:43,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:44,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:45,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:46,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:46,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:47,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:48,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:49,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:50,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:51,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:52,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:52,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:53,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:54,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:55,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:04:55,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:04:55,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:56,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:57,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:58,284 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:04:58,381 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:04:58,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:04:59,381 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:04:59,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:00,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:01,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:02,382 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:02,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:03,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:03,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:04,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:05,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:06,384 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:06,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:07,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:08,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:09,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:09,815 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:10,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:10,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:10,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:11,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:12,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:13,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:14,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:14,965 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:15,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:16,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:17,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:18,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:19,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:20,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:20,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:21,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:22,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:23,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:24,818 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:25,432 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-17 17:05:25,432 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:05:25,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:25,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:25,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:25,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:26,818 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:27,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:27,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:05:27,934 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 with policy now +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:05:27,934 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:05:27,935 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:05:28,166 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/9rtr7aid-media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:05:28,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos +2025-01-17 17:05:28,662 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/572eui1l-media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:05:28,829 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:29,830 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:30,390 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:30,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:31,513 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:31,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:32,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:33,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:34,392 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:34,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:35,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:36,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:36,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:36,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:37,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:05:37,645 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/4a9r2he9-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:05:37,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:38,393 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:38,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:39,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:40,394 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:40,808 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:40,808 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:40,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:41,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:42,331 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:42,395 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:05:42,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:43,832 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:44,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:45,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:46,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:47,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:47,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:48,833 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:49,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:50,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:51,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:52,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:52,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:53,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:54,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:55,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:05:55,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:05:55,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:05:55,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:56,834 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:57,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:57,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:05:58,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:05:59,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:00,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:01,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:02,835 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:03,811 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:03,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:04,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:05,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:06,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:07,837 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:08,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:08,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:09,498 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:06:09,498 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 with policy now +2025-01-17 17:06:09,503 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:06:09,504 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:09,504 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:09,505 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:09,730 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/v56lluot-media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:09,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:10,405 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:10,405 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:10,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:10,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:06:10,681 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/jiv00bf1-media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:06:10,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:10,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:10,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:11,406 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:11,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:12,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:13,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:14,161 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:14,407 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:14,852 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:15,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:16,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:17,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:18,408 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:18,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:19,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:19,934 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:20,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:21,409 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:06:21,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:22,292 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:22,293 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:22,294 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:22,409 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:22,410 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:22,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:23,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:24,410 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:24,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:25,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:25,433 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:06:25,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:25,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:25,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:26,411 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:26,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:27,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:28,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:29,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:30,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:30,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:31,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:32,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:33,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:34,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:35,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:35,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:36,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:37,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:38,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:39,844 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:40,809 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:40,809 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:40,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:40,928 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:41,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:42,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:43,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:44,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:45,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:46,813 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:46,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:47,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:48,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:49,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:50,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:51,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:51,846 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:52,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:53,333 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:06:53,334 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 with policy now +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:06:53,334 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:06:53,335 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:06:53,557 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/hjfo797v-media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,824 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/399xn1bs-media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:06:53,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:54,420 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:54,852 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:55,421 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:55,434 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:06:55,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:06:55,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:06:55,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:56,855 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:56,875 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:06:57,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:58,422 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:06:58,856 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:06:59,853 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:00,856 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:01,855 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:02,423 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:02,423 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:02,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:03,051 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:03,051 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:03,052 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:03,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:03,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:03,752 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/cw6iju6i-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:03,859 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:04,424 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:04,863 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:05,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:06,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:07,463 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:07,864 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:08,426 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:08,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:09,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:10,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:10,810 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:10,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:11,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:12,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:12,966 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:13,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:14,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:15,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:16,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:17,865 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:18,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:18,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:19,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:20,430 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:20,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:21,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:22,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:23,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:23,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:24,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:25,434 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:07:25,810 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:25,811 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:25,867 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:26,867 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:27,868 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:28,868 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:28,963 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:29,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:30,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:31,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:32,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:33,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:34,258 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:34,869 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:35,492 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:07:35,492 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 with policy now +2025-01-17 17:07:35,498 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:07:35,499 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:35,499 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:35,499 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:35,712 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/2uyrxgc1-media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:35,878 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:36,435 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:36,435 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:36,436 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:36,436 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:07:36,719 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tz1fsr13-media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:07:36,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:37,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:38,878 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:39,880 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:40,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:40,437 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:40,814 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:40,814 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:40,883 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:41,913 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:42,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:43,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:44,438 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:44,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:45,191 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:45,438 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:07:45,883 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:46,104 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:07:46,104 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:07:46,105 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:07:46,439 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:46,439 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:07:46,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:47,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:48,440 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:48,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:49,887 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:50,207 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:50,440 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:07:50,887 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:51,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:52,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:53,888 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:54,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:55,435 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:07:55,435 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:07:55,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:07:55,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:07:55,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:56,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:57,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:58,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:07:59,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:00,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:00,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:01,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:02,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:03,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:04,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:05,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:06,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:06,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:07,890 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:08,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:09,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:10,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:10,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:10,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:11,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:11,935 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:12,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:13,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:14,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:15,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:16,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:17,139 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:17,185 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:08:17,185 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:08:17,186 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 with policy now +2025-01-17 17:08:17,186 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:17,186 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:17,187 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:17,403 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/ms63b0z2-media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:08:17,726 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/llcj3vhz-media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:08:17,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:18,450 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:18,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:19,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:20,894 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:21,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:22,452 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:22,861 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:22,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:23,895 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:24,452 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:24,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:25,435 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:08:25,453 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:08:25,733 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/8hgek1ty-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:08:25,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:25,819 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:25,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:26,152 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:26,153 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:26,153 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:26,453 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:26,454 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:26,904 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:27,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:28,297 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:28,454 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:28,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:29,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:30,455 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:30,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:31,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:32,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:33,823 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:33,897 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:34,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:35,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:36,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:37,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:38,823 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:38,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:39,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:40,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:40,812 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:40,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:41,898 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:42,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:43,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:43,918 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:44,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:45,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:46,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:47,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:48,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:49,385 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:49,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:50,463 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:50,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:51,899 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:52,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:53,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:54,464 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:54,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:55,186 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:08:55,436 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:08:55,812 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:08:55,813 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:08:55,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:56,465 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:56,910 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:57,497 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:08:57,497 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 with policy now +2025-01-17 17:08:57,499 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:08:57,500 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:08:57,500 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:08:57,500 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:08:57,711 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tmqsvmfw-media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:57,905 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:58,465 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:58,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:08:58,749 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/qkpxgndv-media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:08:58,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:08:59,466 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:08:59,909 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:00,528 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:00,929 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:01,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:02,467 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:02,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:03,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:04,927 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:05,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:05,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:06,468 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:06,926 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:07,208 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:07,208 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:07,208 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:07,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:07,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:07,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:08,469 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:08,928 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:09,931 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:10,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:10,816 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:10,914 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:10,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:11,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:12,470 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:12,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:13,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:14,922 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:15,923 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:16,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:16,923 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:17,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:18,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:19,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:20,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:21,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:22,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:22,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:23,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:24,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:25,436 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:09:25,816 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:25,816 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:25,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:26,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:27,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:27,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:28,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:29,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:30,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:31,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:32,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:33,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:33,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:34,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:35,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:36,925 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:37,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:38,480 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:38,757 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:09:38,757 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 with policy now +2025-01-17 17:09:38,761 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:09:38,761 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:38,762 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:38,762 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:38,933 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:38,972 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/h0247r19-media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:09:39,754 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/vkbl2cmu-media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:09:39,779 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:39,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:40,481 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:40,825 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:40,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:40,938 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:41,945 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:42,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:43,954 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:44,483 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:44,959 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:45,294 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:45,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:46,254 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:09:46,255 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:09:46,272 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:46,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:09:46,746 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/53x0vfc7-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:09:46,949 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:47,953 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:48,484 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:48,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:49,945 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:50,384 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:50,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:51,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:52,486 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:09:52,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:53,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:54,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:55,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:09:55,437 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:09:55,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:09:55,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:09:55,939 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:56,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:57,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:58,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:09:59,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:00,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:00,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:01,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:02,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:03,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:04,941 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:05,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:05,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:06,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:07,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:08,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:09,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:10,817 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:10,817 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:10,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:10,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:11,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:12,942 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:13,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:14,494 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:14,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:15,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:16,828 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:16,943 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:17,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:18,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:18,599 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:10:18,599 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 with policy now +2025-01-17 17:10:18,602 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:10:18,602 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:10:18,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:10:18,603 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:10:18,833 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/l1gtgiy4-media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:18,948 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:19,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:19,496 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:10:19,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:19,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:10:19,780 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/wmowzvcs-media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:10:19,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:20,497 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:20,960 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:21,836 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:21,961 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:22,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:23,965 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:24,498 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:24,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:25,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:10:25,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:25,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:25,974 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:26,499 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:26,977 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:26,980 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:27,972 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:28,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:29,310 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:10:29,310 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:10:29,311 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:10:29,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:10:29,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:10:29,979 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:30,500 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:30,983 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:31,985 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:32,501 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:32,851 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:32,978 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:33,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:34,502 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:10:34,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:35,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:36,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:37,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:38,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:38,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:39,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:40,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:40,819 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:40,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:41,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:42,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:43,931 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:43,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:44,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:45,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:46,968 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:47,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:48,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:49,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:49,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:50,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:51,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:52,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:53,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:54,838 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:54,969 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:55,437 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:10:55,819 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:10:55,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:10:55,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:56,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:57,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:58,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:10:59,882 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:10:59,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:00,980 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:01,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:11:01,444 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 with policy now +2025-01-17 17:11:01,447 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:11:01,447 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:01,448 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:01,448 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:01,512 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:01,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:11:01,673 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xwjsro3l-media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,964 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/j2674ryu-media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:11:01,972 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:02,513 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:02,979 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:03,982 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:04,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:05,102 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:05,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:06,514 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:06,995 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:07,996 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:08,987 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:09,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:10,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:10,516 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:10,827 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:10,827 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:10,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:11,989 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:12,364 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:12,365 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:12,365 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:12,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:12,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:12,777 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xf278aws-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:12,992 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:14,000 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:14,517 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:15,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:15,245 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:16,014 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:17,011 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:18,012 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:18,519 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:19,011 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:20,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:20,520 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:20,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:21,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:22,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:22,521 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:23,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:24,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:25,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:25,438 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:11:25,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:25,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:25,905 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:26,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:27,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:28,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:29,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:30,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:31,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:31,843 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:32,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:33,006 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:34,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:35,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:36,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:36,844 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:37,007 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:38,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:39,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:40,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:40,820 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:40,820 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:41,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:41,930 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:42,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:43,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:44,008 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:45,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:46,009 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:47,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:47,510 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:47,911 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:11:47,911 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 with policy now +2025-01-17 17:11:47,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:11:47,921 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:47,922 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:47,922 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:48,012 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:48,143 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/tpy9igkc-media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:48,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:48,532 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:11:48,841 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/fwnoxuum-media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:11:49,020 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:50,026 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:50,531 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:51,031 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:52,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:52,532 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:52,621 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:53,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:54,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:55,030 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:55,438 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:11:55,824 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:11:55,824 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:11:56,050 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:56,533 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:57,031 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:57,422 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:11:57,422 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:11:57,423 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:11:57,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:11:57,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:11:58,015 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:11:58,061 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:11:58,534 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:11:59,035 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:00,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:00,535 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:01,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:02,044 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:02,536 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:03,057 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:03,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:04,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:05,037 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:06,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:06,537 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:07,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:08,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:09,038 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:09,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:10,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:10,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:10,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:11,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:12,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:13,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:14,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:14,541 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:15,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:15,453 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:16,039 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:17,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:18,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:19,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:20,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:20,857 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:21,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:22,040 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:23,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:24,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:25,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:25,439 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:12:25,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:25,822 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:25,896 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:26,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:27,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:28,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:29,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:30,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:31,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:31,858 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:32,049 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:32,547 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:33,071 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:33,339 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:12:33,340 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 with policy now +2025-01-17 17:12:33,346 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:12:33,346 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:12:33,347 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:12:33,347 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:33,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:12:33,606 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/43r1mwhm-media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:33,875 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/v9ypo0s8-media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:12:34,080 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:34,548 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:35,086 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:35,549 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:36,072 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:36,884 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:37,092 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:38,078 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:38,550 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:39,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:40,129 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:40,845 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:40,845 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:41,097 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:41,934 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:42,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:42,551 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:43,091 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:43,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:44,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:44,485 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:12:44,485 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:12:44,485 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:12:44,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:44,552 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:44,553 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:12:44,812 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/kzfrmn3i-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:12:45,101 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:46,105 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:46,553 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:47,120 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:47,666 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:48,098 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:48,554 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:49,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:50,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:51,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:52,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:52,555 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:12:52,761 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:53,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:54,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:55,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:55,439 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:12:55,822 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:12:55,823 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:12:56,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:57,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:57,900 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:12:58,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:12:59,103 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:00,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:00,559 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:01,104 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:02,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:03,106 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:03,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:04,106 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:05,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:06,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:07,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:08,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:08,879 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:09,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:10,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:10,563 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:10,824 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:10,824 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:11,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:12,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:13,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:13,974 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:14,107 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:15,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:16,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:17,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:18,108 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:19,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:19,257 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:20,030 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:13:20,030 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 with policy now +2025-01-17 17:13:20,045 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:13:20,046 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:13:20,046 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:13:20,046 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:13:20,114 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:20,258 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/74h79pn1-media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:20,567 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:13:20,848 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/yyc5kh6c-media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:13:21,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:22,128 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:22,568 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:23,117 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:24,123 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:24,568 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:24,797 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:25,125 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:25,440 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:13:25,840 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:25,840 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:26,146 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:27,122 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:28,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:28,540 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:13:28,541 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:13:28,541 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:13:28,570 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:13:29,131 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:30,144 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:30,571 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:30,605 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:31,143 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:32,146 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:33,143 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:34,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:34,572 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:35,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:35,903 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:36,169 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:37,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:38,147 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:38,574 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:39,147 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:40,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:40,826 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:40,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:40,904 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:41,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:42,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:43,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:44,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:44,576 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:45,148 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:46,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:46,493 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:46,577 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:47,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:48,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:48,578 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:13:49,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:50,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:51,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:51,885 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:52,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:53,149 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:54,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:55,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:55,440 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:13:55,826 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:13:55,826 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:13:56,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:56,917 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:13:57,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:58,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:13:59,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:00,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:01,150 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:02,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:02,885 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:03,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:04,151 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:05,174 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:05,712 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:14:05,712 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 with policy now +2025-01-17 17:14:05,724 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:14:05,725 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:05,725 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:05,725 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:05,976 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/cqf2nnqn-media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:06,160 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:06,585 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:06,586 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:14:06,873 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/aiwicnmj-media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:14:07,167 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:07,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:08,189 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:09,164 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:10,186 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:10,587 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:10,830 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:10,831 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:11,178 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:12,190 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:13,167 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:13,509 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:14,182 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:14,588 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:15,181 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:15,589 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:14:15,601 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:15,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:15,602 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:16,182 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:16,589 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:16,590 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:17,207 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:18,189 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:18,590 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:19,210 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:19,223 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:20,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:20,591 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:21,202 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:22,205 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:23,209 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:24,210 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:24,919 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:25,235 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:25,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:14:25,849 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:25,849 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:26,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:26,593 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:27,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:28,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:29,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:30,019 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:30,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:31,218 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:32,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:33,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:34,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:35,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:35,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:36,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:37,219 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:38,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:39,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:40,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:40,831 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:40,831 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:40,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:41,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:42,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:43,220 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:44,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:45,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:46,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:46,908 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:47,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:48,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:49,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:50,221 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:51,222 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:51,909 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:52,222 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:53,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:54,249 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:54,475 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:14:54,475 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 with policy now +2025-01-17 17:14:54,478 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:14:54,478 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:14:54,478 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:14:54,478 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:54,605 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:14:54,699 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/5e087m64-media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:54,968 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/dxwl07yr-media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:14:55,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:55,441 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:14:55,854 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:14:55,855 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:14:56,250 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:56,606 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:14:56,967 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:14:57,283 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:58,272 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:14:59,274 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:00,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:00,607 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:01,257 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:02,258 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:02,262 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:02,608 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:03,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:03,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:03,656 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:03,657 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:03,657 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:03,907 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/q8kjf0e0-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:04,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:04,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:04,609 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:05,285 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:06,285 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:06,610 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:07,274 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:07,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:08,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:08,611 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:09,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:10,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:10,841 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:10,842 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:11,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:12,266 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:12,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:13,267 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:14,267 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:15,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:16,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:17,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:18,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:18,916 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:19,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:20,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:21,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:22,268 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:23,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:24,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:24,523 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:25,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:25,442 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:15:25,842 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:25,842 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:26,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:26,618 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:27,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:28,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:29,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:29,975 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:30,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:31,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:32,269 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:33,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:34,270 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:35,065 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:35,299 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:35,583 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:15:35,583 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 with policy now +2025-01-17 17:15:35,602 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:15:35,602 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:35,602 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:35,603 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:35,622 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:35,622 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:35,623 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:15:35,852 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/9yeyz679-media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:36,156 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/a0xfvtn6-media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:15:36,312 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:36,623 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:37,282 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:38,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:39,329 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:40,311 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:40,329 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:40,624 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:40,850 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:40,850 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:41,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:42,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:43,323 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:44,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:44,626 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:45,301 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:45,452 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:46,309 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:47,310 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:48,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:49,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:50,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:50,940 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:51,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:52,335 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:52,629 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:52,629 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:15:52,714 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:15:52,714 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:15:52,714 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:15:53,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:53,630 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:15:54,344 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:54,630 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:55,332 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:55,442 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:15:55,892 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:15:55,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:15:56,052 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:15:56,326 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:57,317 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:58,317 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:15:58,632 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:15:59,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:00,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:01,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:01,936 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:02,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:03,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:04,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:05,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:06,318 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:07,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:07,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:08,319 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:09,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:10,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:10,848 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:10,848 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:11,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:12,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:12,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:13,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:14,320 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:15,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:16,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:17,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:17,937 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:18,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:19,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:20,321 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:21,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:22,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:22,938 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:23,322 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:24,323 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:24,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:24,906 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:16:24,906 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 with policy now +2025-01-17 17:16:24,924 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:16:24,925 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:16:24,925 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:16:24,925 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:16:25,120 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/6hpol3vb-media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:25,328 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:25,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:16:25,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:16:25,643 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:25,644 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:16:25,866 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:25,866 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:25,927 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/mwhuv8dw-media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:16:26,370 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:26,644 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:27,341 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:27,995 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:28,334 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:29,331 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:30,338 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:30,645 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:31,344 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:32,360 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:32,646 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:33,365 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:33,769 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:34,370 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:35,356 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:35,647 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:16:35,768 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:16:35,768 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:16:35,768 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:16:36,360 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:36,648 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:16:37,349 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:37,648 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:38,368 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:39,333 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:39,377 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:39,649 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:40,368 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:40,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:40,894 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:41,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:41,650 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:42,389 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:43,388 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:44,421 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:44,991 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:45,383 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:46,385 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:47,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:48,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:49,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:49,653 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:16:50,214 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:50,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:51,371 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:52,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:53,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:54,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:55,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:55,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:16:55,444 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:16:55,876 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:16:55,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:16:56,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:57,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:58,372 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:16:59,374 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:00,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:00,947 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:01,374 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:02,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:03,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:04,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:05,375 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:05,948 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:06,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:07,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:08,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:09,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:09,662 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:10,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:10,876 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:10,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:11,022 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:11,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:12,376 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:13,377 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:14,387 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:15,238 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:17:15,238 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 with policy now +2025-01-17 17:17:15,252 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:17:15,253 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:17:15,253 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:17:15,253 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:17:15,393 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:15,461 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/k34rrn1p-media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:15,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:17:16,312 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/xg3mrrkk-media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:17:16,406 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:16,665 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:17,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:17,409 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:17,666 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:18,440 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:19,415 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:20,440 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:21,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:21,667 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:22,023 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:22,412 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:23,429 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:23,668 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:24,425 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:25,427 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:25,443 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:17:25,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:25,889 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:26,445 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:27,429 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:27,970 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:28,448 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:29,478 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:29,671 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:17:29,829 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:17:29,830 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:17:29,830 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:17:29,938 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/81nzpym7-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:17:30,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:30,671 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:17:31,455 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:31,672 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:32,451 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:32,978 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:33,448 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:34,468 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:35,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:35,673 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:36,470 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:37,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:38,474 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:38,966 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:39,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:40,475 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:40,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:40,892 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:41,473 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:42,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:43,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:43,677 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:17:44,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:44,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:45,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:46,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:47,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:48,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:49,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:49,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:50,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:51,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:52,456 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:53,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:54,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:54,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:17:55,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:17:55,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:55,889 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:17:55,889 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:17:56,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:57,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:58,457 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:17:59,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:00,049 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:00,458 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:01,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:02,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:03,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:04,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:05,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:05,958 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:06,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:07,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:08,459 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:09,483 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:10,490 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:10,561 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:18:10,561 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 with policy now +2025-01-17 17:18:10,586 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:18:10,587 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:18:10,587 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:18:10,587 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:10,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:18:10,799 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/b9qauszt-media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:10,891 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:10,891 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:10,985 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:11,088 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/povne3nd-media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:18:11,495 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:11,689 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:12,498 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:12,690 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:13,520 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:13,690 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:14,491 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:15,492 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:15,691 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:16,503 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:16,964 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:17,495 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:18,507 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:19,527 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:19,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:19,891 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:18:19,891 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:18:19,891 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:18:20,505 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:20,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:18:20,693 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:18:21,524 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:21,694 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:22,069 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:22,520 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:23,516 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:23,695 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:24,508 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:25,444 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:18:25,518 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:25,695 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:25,906 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:25,906 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:26,577 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:27,533 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:28,044 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:28,549 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:29,534 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:30,541 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:31,568 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:32,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:33,080 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:33,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:34,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:35,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:35,700 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:18:36,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:37,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:38,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:38,975 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:39,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:40,551 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:40,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:40,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:41,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:42,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:43,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:44,027 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:44,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:45,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:46,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:47,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:48,552 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:49,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:49,976 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:50,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:51,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:52,553 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:53,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:54,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:54,976 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:18:55,445 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:18:55,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:55,893 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:18:55,893 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:18:56,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:57,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:58,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:18:59,554 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:00,314 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:00,571 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:01,560 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:01,616 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:19:01,616 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 with policy now +2025-01-17 17:19:01,657 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:19:01,657 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:01,658 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:01,658 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:01,711 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:01,711 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:01,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:01,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:19:01,847 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/vnkkhudp-media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:02,161 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/bvgjirbq-media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:19:02,603 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:03,596 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:03,712 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:04,601 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:04,713 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:05,343 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:05,604 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:06,612 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:07,605 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:07,714 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:08,657 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:09,608 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:09,953 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:09,953 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:09,954 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:10,637 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:10,715 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:19:10,716 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:10,921 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:10,922 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:10,992 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:11,684 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:11,716 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:12,640 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:13,642 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:13,717 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:14,637 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:15,702 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:15,718 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:16,002 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:16,675 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:17,685 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:18,705 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:19,699 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:20,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:21,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:21,720 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:21,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:22,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:23,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:23,721 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:24,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:25,445 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:19:25,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:25,901 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:25,902 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:26,690 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:26,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:27,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:28,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:29,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:30,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:31,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:32,691 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:32,981 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:33,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:34,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:35,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:36,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:37,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:37,982 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:38,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:39,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:40,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:40,902 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:40,902 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:41,692 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:42,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:43,025 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:43,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:44,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:45,693 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:46,694 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:47,737 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:48,057 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:48,677 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:19:48,677 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 with policy now +2025-01-17 17:19:48,683 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:19:48,683 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:19:48,684 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:19:48,684 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:19:48,732 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:19:48,732 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:48,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval +2025-01-17 17:19:48,748 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:48,897 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/zpvjv7e8-media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:49,164 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/yofnmct4-media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:19:49,719 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:49,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:50,733 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:50,739 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:51,734 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:51,762 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:52,734 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:53,465 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:53,735 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:53,743 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:54,744 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:55,446 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:19:55,747 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:55,917 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:19:55,918 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:19:56,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:57,736 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:19:57,786 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:58,774 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:19:59,070 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:19:59,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:00,738 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:00,794 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:01,019 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:01,019 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:01,019 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:01,738 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:01,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:01,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:01,796 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 17:20:01,796 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 17:20:02,191 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:20:02,295 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpz47qb_pz +2025-01-17 17:20:02,647 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-17 17:20:02,656 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: partial_history +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:02,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media +2025-01-17 17:20:02,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval +2025-01-17 17:20:02,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table +2025-01-17 17:20:02,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:02,868 INFO SenderThread:2529582 [sender.py:send_request_log_artifact():1525] logged artifact run-bgznqd9b-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTM5ODExNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTY5NDg1Mw==', 'latestArtifact': None}} +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:20:02,868 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json with policy now +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:02,868 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,869 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,869 DEBUG SenderThread:2529582 [sender.py:send():379] send: telemetry +2025-01-17 17:20:02,869 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,873 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,873 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,874 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,874 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,874 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,874 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,875 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,875 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,876 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,876 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,876 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,877 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,877 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,877 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,877 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,878 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,878 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,878 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,878 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:02,879 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:02,879 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: log_artifact +2025-01-17 17:20:03,080 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/0te42e0p-media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:03,125 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/haat8bgm-media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:03,127 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/c056iigq-events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:03,300 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0g30cvrc +2025-01-17 17:20:03,347 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/i1nwfp2t-media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:03,720 INFO SenderThread:2529582 [sender.py:send_request_log_artifact():1525] logged artifact run-bgznqd9b-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ0OTM5ODEyNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0MTY5NDg1Ng==', 'latestArtifact': None}} +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send():379] send: files +2025-01-17 17:20:03,720 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json with policy now +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:03,720 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:03,721 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:03,739 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:03,740 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:03,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:03,976 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /tmp/tmpafzmvxpxwandb/zv8pew0g-media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:04,725 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:04,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:05,741 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:20:05,741 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:05,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:06,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:07,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:08,770 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:09,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:10,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:10,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:10,902 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:20:10,903 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:20:11,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:12,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:13,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:14,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:15,017 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:15,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:16,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:17,771 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:18,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:19,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:20,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:21,004 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:21,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:22,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:23,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:24,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:25,446 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:20:25,772 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:25,903 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: stop_status +2025-01-17 17:20:25,903 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: stop_status +2025-01-17 17:20:26,005 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:26,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:27,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:28,773 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:29,237 DEBUG SenderThread:2529582 [sender.py:send():379] send: exit +2025-01-17 17:20:29,237 INFO SenderThread:2529582 [sender.py:send_exit():586] handling exit code: 0 +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_exit():588] handling runtime: 963 +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_exit():594] send defer +2025-01-17 17:20:29,238 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,238 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-17 17:20:29,238 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,238 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 1 +2025-01-17 17:20:29,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-17 17:20:29,239 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-17 17:20:29,239 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 2 +2025-01-17 17:20:29,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-17 17:20:29,239 INFO HandlerThread:2529582 [system_monitor.py:finish():203] Stopping system monitor +2025-01-17 17:20:29,239 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-17 17:20:29,240 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined cpu monitor +2025-01-17 17:20:29,240 DEBUG SystemMonitor:2529582 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-17 17:20:29,240 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined disk monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined gpu monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined memory monitor +2025-01-17 17:20:29,292 INFO HandlerThread:2529582 [interfaces.py:finish():200] Joined network monitor +2025-01-17 17:20:29,293 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,293 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-17 17:20:29,293 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 3 +2025-01-17 17:20:29,293 DEBUG SenderThread:2529582 [sender.py:send():379] send: stats +2025-01-17 17:20:29,293 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,293 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-17 17:20:29,294 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:29,294 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-17 17:20:29,294 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 4 +2025-01-17 17:20:29,294 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:29,294 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-17 17:20:29,754 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:31,755 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:36,028 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: keepalive +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,029 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-17 17:20:36,029 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 5 +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:36,029 DEBUG SenderThread:2529582 [sender.py:send():379] send: history +2025-01-17 17:20:36,029 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,030 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: summary_record +2025-01-17 17:20:36,030 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-17 17:20:36,031 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:36,040 DEBUG SenderThread:2529582 [sender.py:send():379] send: summary +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-17 17:20:36,041 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-17 17:20:36,041 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 6 +2025-01-17 17:20:36,041 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,041 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-17 17:20:36,041 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 7 +2025-01-17 17:20:36,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: status_report +2025-01-17 17:20:36,042 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:36,042 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-17 17:20:36,042 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:36,042 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-17 17:20:36,239 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:36,757 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:37,989 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 8 +2025-01-17 17:20:37,989 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:37,990 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:37,990 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-17 17:20:37,990 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:37,990 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-17 17:20:37,990 INFO SenderThread:2529582 [job_builder.py:build():440] Attempting to build job artifact +2025-01-17 17:20:37,993 INFO SenderThread:2529582 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-17 17:20:37,994 INFO SenderThread:2529582 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-17 17:20:38,010 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 9 +2025-01-17 17:20:38,010 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:38,010 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-17 17:20:38,011 DEBUG SenderThread:2529582 [sender.py:send():379] send: artifact +2025-01-17 17:20:38,240 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:38,465 INFO wandb-upload_1:2529582 [upload_job.py:push():85] Skipped uploading /tmp/tmpb4lafne_/wandb-job.json +2025-01-17 17:20:38,468 INFO wandb-upload_3:2529582 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp72azu4yv +2025-01-17 17:20:38,583 INFO wandb-upload_0:2529582 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp87v6_dh3 +2025-01-17 17:20:38,758 INFO Thread-12 :2529582 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ0OTQwMTA5Mw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ0OTM2NTUwOA==', 'versionIndex': 444}}} +2025-01-17 17:20:39,176 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-17 17:20:39,176 INFO SenderThread:2529582 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml config.yaml +2025-01-17 17:20:39,759 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json wandb-summary.json +2025-01-17 17:20:39,760 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt requirements.txt +2025-01-17 17:20:39,761 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-metadata.json wandb-metadata.json +2025-01-17 17:20:39,762 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 events.out.tfevents.1737151465.leonmkim-ROG-Strix-G15CS-G15CS.2529391.0 +2025-01-17 17:20:39,763 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch upstream_diff_0f018502ce3c0d2ad4dfb21f3d7778b9e3db3a78.patch +2025-01-17 17:20:39,763 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log output.log +2025-01-17 17:20:39,765 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/diff.patch diff.patch +2025-01-17 17:20:39,766 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 media/videos/eval/18_eval_18_8f12dd2579d2c81aa354.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 media/videos/eval/5_eval_5_7ecf6a30ef0ba5b1a068.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 media/videos/eval/7_eval_7_38ea89fb859f97aaeb81.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 media/videos/eval/17_eval_17_0d21e9b6a3b8f35c0680.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 media/videos/eval/10_eval_10_9dd459ed3c989dccd23a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 media/videos/eval/14_eval_14_6742f7ac7cbb27a3fbe8.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 media/videos/eval/1_eval_1_7525ad080ceebbb86e1e.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 media/videos/eval/3_eval_3_651f1db6e4859b8c80d6.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 media/videos/eval/6_eval_6_9ad60ce20c1e905ff49a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 media/videos/eval/19_eval_19_c5613e6972fd6e3202ca.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 media/videos/eval/16_eval_16_e9f643ba3fec49b0d46e.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 media/videos/eval/2_eval_2_8ab171c7134ecf8bdca9.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 media/videos/eval/13_eval_13_85a3a26bb1dec2c3b14a.mp4 +2025-01-17 17:20:39,767 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 media/videos/eval/8_eval_8_33454563ddfcbdb93106.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 media/videos/eval/9_eval_9_042356fae54e7c9746ea.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 media/videos/eval/15_eval_15_deab99abf59c9ff5fce7.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 media/videos/eval/0_eval_0_cd6283fb51b44e5730f0.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 media/videos/eval/4_eval_4_db56d42ec91352f013c0.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 media/videos/eval/12_eval_12_b8267a8b4713015b00d7.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 media/videos/eval/11_eval_11_8aa51a8297fd64b89173.mp4 +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json media/table/eval/total_success_rate_ci_21_99066e668ec75c97d0dd.table.json +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json media/table/eval/success_rate_ci_20_203f9a9e17ed51251fdd.table.json +2025-01-17 17:20:39,768 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 10 +2025-01-17 17:20:39,768 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:39,768 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:39,768 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-17 17:20:39,769 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:39,769 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-17 17:20:39,769 INFO SenderThread:2529582 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 17:20:39,953 INFO wandb-upload_3:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/requirements.txt +2025-01-17 17:20:39,955 INFO wandb-upload_2:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/config.yaml +2025-01-17 17:20:39,978 INFO wandb-upload_1:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/wandb-summary.json +2025-01-17 17:20:39,983 INFO wandb-upload_0:2529582 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/files/output.log +2025-01-17 17:20:40,183 INFO Thread-11 (_thread_body):2529582 [sender.py:transition_state():614] send defer: 11 +2025-01-17 17:20:40,183 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,183 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-17 17:20:40,183 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,183 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-17 17:20:40,183 INFO SenderThread:2529582 [file_pusher.py:join():175] waiting for file pusher +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 12 +2025-01-17 17:20:40,187 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,187 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-17 17:20:40,187 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-17 17:20:40,187 INFO SenderThread:2529582 [file_stream.py:finish():601] file stream finish called +2025-01-17 17:20:40,240 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [file_stream.py:finish():605] file stream finish is done +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 13 +2025-01-17 17:20:40,402 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,402 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,402 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-17 17:20:40,402 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-17 17:20:40,402 INFO SenderThread:2529582 [sender.py:transition_state():614] send defer: 14 +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: defer +2025-01-17 17:20:40,403 INFO HandlerThread:2529582 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send():379] send: final +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send():379] send: footer +2025-01-17 17:20:40,403 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: defer +2025-01-17 17:20:40,403 INFO SenderThread:2529582 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,403 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: server_info +2025-01-17 17:20:40,404 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,404 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: server_info +2025-01-17 17:20:40,405 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-17 17:20:40,405 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: get_summary +2025-01-17 17:20:40,406 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-17 17:20:40,407 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-17 17:20:40,460 DEBUG SenderThread:2529582 [sender.py:send_request():406] send_request: poll_exit +2025-01-17 17:20:40,460 INFO MainThread:2529582 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-17 17:20:40,461 INFO MainThread:2529582 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-17 17:20:40,461 INFO MainThread:2529582 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-17 17:20:40,461 DEBUG HandlerThread:2529582 [handler.py:handle_request():158] handle_request: shutdown +2025-01-17 17:20:40,462 INFO HandlerThread:2529582 [handler.py:finish():882] shutting down handler +2025-01-17 17:20:41,406 INFO WriterThread:2529582 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [sender.py:finish():1615] shutting down sender +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [file_pusher.py:finish():169] shutting down file pusher +2025-01-17 17:20:41,460 INFO SenderThread:2529582 [file_pusher.py:join():175] waiting for file pusher diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug.log b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..95e34b4e8053f2f0fd443cddd24d9353d86c689e --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Configure stats pid to 2529391 +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-17 17:04:25,116 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/settings +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug.log +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/logs/debug-internal.log +2025-01-17 17:04:25,117 INFO MainThread:2529391 [wandb_init.py:init():569] calling init triggers +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '3456_2', 'wandb_notes': '3456_2_req_3727_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': None}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b'} +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():619] starting backend +2025-01-17 17:04:25,118 INFO MainThread:2529391 [wandb_init.py:init():623] setting up manager +2025-01-17 17:04:25,122 INFO MainThread:2529391 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-17 17:04:25,124 INFO MainThread:2529391 [wandb_init.py:init():631] backend started and connected +2025-01-17 17:04:25,137 INFO MainThread:2529391 [wandb_init.py:init():720] updated telemetry +2025-01-17 17:04:25,143 INFO MainThread:2529391 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-17 17:04:25,377 INFO MainThread:2529391 [wandb_run.py:_on_init():2435] communicating current version +2025-01-17 17:04:25,418 INFO MainThread:2529391 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-17 17:04:25,419 INFO MainThread:2529391 [wandb_init.py:init():804] starting run threads in backend +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_console_start():2413] atexit reg +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-17 17:04:25,808 INFO MainThread:2529391 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-17 17:04:25,810 INFO MainThread:2529391 [wandb_init.py:init():847] run started, returning control to user process +2025-01-17 17:04:25,810 INFO MainThread:2529391 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/bgznqd9b/tb, True +2025-01-17 17:04:32,713 INFO MainThread:2529391 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-17 17:20:41,462 WARNING MsgRouterThr:2529391 [router.py:message_loop():77] message_loop has been closed diff --git a/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8f872591c1847e8dbee558cde12bb5bb498a9216 --- /dev/null +++ b/bgznqd9b/wandb/run-20250117_170425-bgznqd9b/run-bgznqd9b.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e00677bfe05e5e7fcec5b62655934492db85e88a9ba4405c9dc1b020342bd0e +size 230350 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c40cb6a8b31b5790fd4caaa172903ccc53d5a4f --- /dev/null +++ b/config.yaml @@ -0,0 +1,573 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 2 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '3456_2' +wandb_notes: 3456_2_req_3727_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_1_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/l57ht4md/.hydra/config.yaml b/l57ht4md/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fd3a0859b246c9db42fb2b4e2f15f3160aa49a5 --- /dev/null +++ b/l57ht4md/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '3456_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/l57ht4md/.hydra/hydra.yaml b/l57ht4md/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888677e9a3f293fc42fcf36dc4d2cafb003f2f93 --- /dev/null +++ b/l57ht4md/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/3456_2/l57ht4md + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/l57ht4md/.hydra/overrides.yaml b/l57ht4md/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/l57ht4md/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/l57ht4md/eval_policy.log b/l57ht4md/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..9a8528a9c499cca590f966c02f6a8055a01618f1 --- /dev/null +++ b/l57ht4md/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-17 17:04:17,627][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-17 17:04:17,630][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-17 17:04:20,702][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a507be07fb307b1c27e0ba9277de3c8d051f4fb --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28ffc0cef2b67637fa6eee91035cdd938ccf89e0bc95f3b5079e68637e0ef9f1 +size 909989782 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..35f93e13f55c10a2208c6daa0912ae0dbe133052 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6c0d1e1d2ad63a44054ff58c70c63c8b0195e1125395b37954bd9d77cbf4f08 +size 909989782 diff --git a/snapshot_12499.pt b/snapshot_12499.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f30ba8c84c8900af862a3b3a0df482ca5f0a9d7 --- /dev/null +++ b/snapshot_12499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e37223b62199d7d18691054e26ac8c419447c4f7d2709d34db9b02e5c62693ce +size 909989782 diff --git a/snapshot_12999.pt b/snapshot_12999.pt new file mode 100644 index 0000000000000000000000000000000000000000..79e041f79c60ae5ace7a09a3e1fe88805adea963 --- /dev/null +++ b/snapshot_12999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93e5c1d90c305608c405a54465c4e89d2818014e96a5b254d5408d872bc1cc6a +size 909989782 diff --git a/snapshot_13249.pt b/snapshot_13249.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d02c344247d03f64ff049d03aa66248b4f90794 --- /dev/null +++ b/snapshot_13249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:001899b22c278eba9bc6052f8083c535f21010201242ec96fb1bdbb6aab50f0b +size 909989782 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..93aa3a2cfbbc4f5baea76f7071e01670a004b2c0 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2fe74a791efa95ad6c2c861a2d5da30d095415474c2338a10e4e5e53746db85 +size 909989782 diff --git a/snapshot_13749.pt b/snapshot_13749.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5af4f6e7c425d8ae5eb993328950c384ee49dcb --- /dev/null +++ b/snapshot_13749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9561079daff7ea29c56c359585d3f164cf82ff950b5338e4ba86ca855b2dd264 +size 909989782 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a0fe1b3e47caadf953e347cdadb7a94cd175221 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c029d7d81d42636fd06b34842d85da4e4f490fb627ebc80e9e4ccd519f79a985 +size 909989782 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb1951e96d1d001bf5903ad7468339005fc7c8e --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7cf29dc8ace69c0fc69f44194b5b91256715ebe93cd83356e48fe4c188932d5 +size 909989782 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..9eedd861e959e56a90ecfffe134376f74b7a66b5 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:663a22d902e5ee136566b326957635213ea02b4f691047762508ca13dcb2c085 +size 909989782 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..10c1966f7d5e1aeb2f6ef610fd4d691f3ea5b2bc --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4c25e3ec87b1615ee3954dc403700443e606d6a9039380ed7d2f5bd65d96b9 +size 909989782 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4675f6a3062d134c5952be332ac40409aa9ecf68 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a413f738df22d8679b45896309642012468056426f81266aa1a4a00bdd3773ad +size 909989782 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..31a0169e9323c8bd17937142fd17737f9c61bed7 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d326d8f395f507065769b0276f431c0da6b0887c1e3b063f1a8e0a265c57a980 +size 909989782 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fb1951e96d1d001bf5903ad7468339005fc7c8e --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7cf29dc8ace69c0fc69f44194b5b91256715ebe93cd83356e48fe4c188932d5 +size 909989782