diff --git a/.gitattributes b/.gitattributes index 6455a824387b10f46e38d3d5a78604cf75785d59..98f5e9852a378cd942df215b514ecfd13090be76 100644 --- a/.gitattributes +++ b/.gitattributes @@ -393,3 +393,57 @@ opf0nkwg/episode_rosbags/episode_5_2025-01-07-22-33-34.bag filter=lfs diff=lfs m opf0nkwg/episode_rosbags/episode_12_2025-01-07-22-38-52.bag filter=lfs diff=lfs merge=lfs -text rgxendfg/episode_rosbags/episode_0_2025-01-07-20-55-05.bag filter=lfs diff=lfs merge=lfs -text opf0nkwg/episode_rosbags/episode_11_2025-01-07-22-38-07.bag filter=lfs diff=lfs merge=lfs -text +z6f2akbj/episode_rosbags/episode_3_2025-01-07-21-02-51.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/episode_rosbags/episode_18_2025-01-07-23-04-16.bag filter=lfs diff=lfs merge=lfs -text +rgxendfg/episode_rosbags/episode_1_2025-01-07-20-55-45.bag filter=lfs diff=lfs merge=lfs -text +rgxendfg/episode_rosbags/episode_3_2025-01-07-20-57-21.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/episode_rosbags/episode_17_2025-01-07-23-03-22.bag filter=lfs diff=lfs merge=lfs -text +rgxendfg/episode_rosbags/episode_2_2025-01-07-20-56-31.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/episode_rosbags/episode_16_2025-01-07-23-02-28.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/episode_rosbags/episode_7_2025-01-07-22-54-58.bag filter=lfs diff=lfs merge=lfs -text +z6f2akbj/episode_rosbags/episode_1_2025-01-07-21-00-47.bag filter=lfs diff=lfs merge=lfs -text +z6f2akbj/episode_rosbags/episode_2_2025-01-07-21-01-29.bag filter=lfs diff=lfs merge=lfs -text +z6f2akbj/episode_rosbags/episode_0_2025-01-07-20-59-59.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/19_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/12_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/8_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/7_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/13_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/9_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/18_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/6_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/11_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/15_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/5_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/16_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/17_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/14_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/10_eval.mp4 filter=lfs diff=lfs merge=lfs -text +bmk18y9z/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jqiikotd/episode_rosbags/episode_1_2025-01-07-17-24-24.bag filter=lfs diff=lfs merge=lfs -text +bmk18y9z/episode_rosbags/episode_13_2025-01-07-22-59-48.bag filter=lfs diff=lfs merge=lfs -text diff --git a/4fmihp69/episode_rosbags/cam_tf_world.npy b/4fmihp69/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/4fmihp69/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/4fmihp69/eval_robot.log b/4fmihp69/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bece4e57c59c1a3ca39500bf6032ff1aaefee8c7 --- /dev/null +++ b/4fmihp69/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 22:06:04,710][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 22:06:04,713][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/bmk18y9z/.hydra/config.yaml b/bmk18y9z/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a56df077cebbbb54879d37edcc83ff7d1e7c8eb --- /dev/null +++ b/bmk18y9z/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 20 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1002_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bmk18y9z/.hydra/hydra.yaml b/bmk18y9z/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2e81eaf8e955df7a859881a22889251f7a8f6f5 --- /dev/null +++ b/bmk18y9z/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bmk18y9z/.hydra/overrides.yaml b/bmk18y9z/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bmk18y9z/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bmk18y9z/episode_rosbags/aligned_depth_to_color_K.npy b/bmk18y9z/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bmk18y9z/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bmk18y9z/episode_rosbags/color_K.npy b/bmk18y9z/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/bmk18y9z/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/bmk18y9z/episode_rosbags/episode_13_2025-01-07-22-59-48.bag b/bmk18y9z/episode_rosbags/episode_13_2025-01-07-22-59-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..04bf06070d692cad276dd11cdc27c9f4b33d0ea9 --- /dev/null +++ b/bmk18y9z/episode_rosbags/episode_13_2025-01-07-22-59-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25e4086d5e03419227379cc2bc3e21469e986f00f5680e64e19446f8742fb04c +size 1409653577 diff --git a/bmk18y9z/episode_rosbags/episode_16_2025-01-07-23-02-28.bag b/bmk18y9z/episode_rosbags/episode_16_2025-01-07-23-02-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..69b22e40413bad7a8202f4a3f2bce7784108f4fd --- /dev/null +++ b/bmk18y9z/episode_rosbags/episode_16_2025-01-07-23-02-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b98d6aa34e5b888ad9aab3bd7314758e113cfea4ccb7b1382cda981340922be8 +size 1418364402 diff --git a/bmk18y9z/episode_rosbags/episode_17_2025-01-07-23-03-22.bag b/bmk18y9z/episode_rosbags/episode_17_2025-01-07-23-03-22.bag new file mode 100644 index 0000000000000000000000000000000000000000..66b6b2490fe294f35d09399c0f64d3c5cff2d26f --- /dev/null +++ b/bmk18y9z/episode_rosbags/episode_17_2025-01-07-23-03-22.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45bc38ac0ba3f41bda4ccc08e1a88329b587d2c71be45cb020398be8f07fc66f +size 1411758619 diff --git a/bmk18y9z/episode_rosbags/episode_18_2025-01-07-23-04-16.bag b/bmk18y9z/episode_rosbags/episode_18_2025-01-07-23-04-16.bag new file mode 100644 index 0000000000000000000000000000000000000000..54e675c24fdaed34292678b76fd9de049bf86cf8 --- /dev/null +++ b/bmk18y9z/episode_rosbags/episode_18_2025-01-07-23-04-16.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b19d3b92e0a8c293278bb3265a29ea7490390974b46e712a724c6937ec2ab9d1 +size 1410935604 diff --git a/bmk18y9z/episode_rosbags/episode_7_2025-01-07-22-54-58.bag b/bmk18y9z/episode_rosbags/episode_7_2025-01-07-22-54-58.bag new file mode 100644 index 0000000000000000000000000000000000000000..8adff797dc270cffbc3553c8e059daced76bb069 --- /dev/null +++ b/bmk18y9z/episode_rosbags/episode_7_2025-01-07-22-54-58.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07523dfe92dcba57576a4e6a25564ff6fc01d1309c3231879008ef99d37feb0 +size 1413871866 diff --git a/bmk18y9z/eval_video/0_eval.mp4 b/bmk18y9z/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..076d3a037c337c0b15364b43d24198c548f6c98e --- /dev/null +++ b/bmk18y9z/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a7acf7610c7a7f0d62e8d433ce287a8947ce47c8f1195c93b999c25ecce79b3 +size 1138909 diff --git a/bmk18y9z/eval_video/10_eval.mp4 b/bmk18y9z/eval_video/10_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bb0923bdbe831fdca211a9dbdaf558bd776d297 --- /dev/null +++ b/bmk18y9z/eval_video/10_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5570816ad9da40a43e521c2692fbf3c896a7868112e102639b1f803807b568 +size 1278356 diff --git a/bmk18y9z/eval_video/11_eval.mp4 b/bmk18y9z/eval_video/11_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40216455fa2cb1ff1e6f371b61316088d0453eee --- /dev/null +++ b/bmk18y9z/eval_video/11_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a606d9c48c91655df31c54ec48b57edcb5ae66c9e0e72e08d0040360683cbca7 +size 1399219 diff --git a/bmk18y9z/eval_video/12_eval.mp4 b/bmk18y9z/eval_video/12_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57babdb07ee97e0ebe6c5ec0a987786bd79035f3 --- /dev/null +++ b/bmk18y9z/eval_video/12_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a187eeb09299696aa2a1651b0c5885f8d4888b203fafb6872a4451f7c71ffef +size 1427340 diff --git a/bmk18y9z/eval_video/13_eval.mp4 b/bmk18y9z/eval_video/13_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8eca45e01bc4ce3cca52ee13d4aa08b6e83ba697 --- /dev/null +++ b/bmk18y9z/eval_video/13_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9efca5d983ee670cf985545f839f820290fa9648c2efe9805271496b3e2afe5c +size 1205126 diff --git a/bmk18y9z/eval_video/14_eval.mp4 b/bmk18y9z/eval_video/14_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0c7a896759ae2d671156dabd4247f773434c5e2 --- /dev/null +++ b/bmk18y9z/eval_video/14_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffa92cd8767b9ad59d685b39c030e4373019479e38dcd00037ea91346b07353d +size 1373307 diff --git a/bmk18y9z/eval_video/15_eval.mp4 b/bmk18y9z/eval_video/15_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51ee392298feb5331cadf997acd64a97ea5d487a --- /dev/null +++ b/bmk18y9z/eval_video/15_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af536578f4476ce23195e3698f30ff9f31c2321e1d8a9efb7a5d698910722d35 +size 1358907 diff --git a/bmk18y9z/eval_video/16_eval.mp4 b/bmk18y9z/eval_video/16_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b38ebf7122b784a810a6dffd4375edd8423b261e --- /dev/null +++ b/bmk18y9z/eval_video/16_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db31934236335d9e75e4f0c34cec84e72d542118c7c4ee6ce4f5de77075484b3 +size 1444676 diff --git a/bmk18y9z/eval_video/17_eval.mp4 b/bmk18y9z/eval_video/17_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67fbebd68e970c314e3990990dcb4c40e6853146 --- /dev/null +++ b/bmk18y9z/eval_video/17_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0293b276af8b7ee82ad1af37fc416a012c67d2159e42b28d61210c141cd56c7 +size 1445556 diff --git a/bmk18y9z/eval_video/18_eval.mp4 b/bmk18y9z/eval_video/18_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b19802a6ade1903a09c0d86e7e87bd75d0e3c929 --- /dev/null +++ b/bmk18y9z/eval_video/18_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:205c297dae90682b6f426123d9088c72777d0add28b23f8d2ff0181423bf5020 +size 1459635 diff --git a/bmk18y9z/eval_video/19_eval.mp4 b/bmk18y9z/eval_video/19_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a091605756e16e95fb53840d565df23e5b73573 --- /dev/null +++ b/bmk18y9z/eval_video/19_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b3cf56f899624bcd38b38ef615d544991af40aab002b25fe8e00be0095a8ca +size 1191669 diff --git a/bmk18y9z/eval_video/1_eval.mp4 b/bmk18y9z/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a117341396ed7587e221304261e7cef9e08520f --- /dev/null +++ b/bmk18y9z/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6105fa800baa66794d18fe8647635294b8fd467674d9a85bfd6647039cf1293d +size 1193600 diff --git a/bmk18y9z/eval_video/2_eval.mp4 b/bmk18y9z/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..826803eff02e64384ed8b2525da0dfecc90d3249 --- /dev/null +++ b/bmk18y9z/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92292561966baf2ee3899d136adc87f373197924f837a6a3dddb13dd7dfc730 +size 1143421 diff --git a/bmk18y9z/eval_video/3_eval.mp4 b/bmk18y9z/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2474d7bc0b6ce608db49f3ba5cfb4bd40e89be80 --- /dev/null +++ b/bmk18y9z/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4978aa6435b32ecf67343f7f22cb866f6a82d85d5d1892976fb019fffb095a08 +size 1200135 diff --git a/bmk18y9z/eval_video/4_eval.mp4 b/bmk18y9z/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dce9f5ab09fe94cbe396bd0ff885e6be04a18b2e --- /dev/null +++ b/bmk18y9z/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf6c04acd951daf42e0863e775e650eb62d0fba1c2c91368af151a2718d6a42 +size 1247071 diff --git a/bmk18y9z/eval_video/5_eval.mp4 b/bmk18y9z/eval_video/5_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5aaa39f3a82a24e23726ebdf55c992fcbad75e5a --- /dev/null +++ b/bmk18y9z/eval_video/5_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c452dd651b0df318063859d846c3dd0082130d16fe37e52f60da91cffdecb788 +size 1198593 diff --git a/bmk18y9z/eval_video/6_eval.mp4 b/bmk18y9z/eval_video/6_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc68d7d0db83f560b28dce118b8ea16dbc0c2f47 --- /dev/null +++ b/bmk18y9z/eval_video/6_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a40bb3e24616a11f81cf8f8cb531134b91ca09b4d31f5c66d26081f8a597697a +size 1243138 diff --git a/bmk18y9z/eval_video/7_eval.mp4 b/bmk18y9z/eval_video/7_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa48398d2adedf7528a12830d290d9f235981eb7 --- /dev/null +++ b/bmk18y9z/eval_video/7_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1822051540a8ae17e31542973fd9483b3e04543b61fa35aa2e112db4e7930691 +size 1216136 diff --git a/bmk18y9z/eval_video/8_eval.mp4 b/bmk18y9z/eval_video/8_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1769b65b894045d3c4a43de78f4468d40f69db77 --- /dev/null +++ b/bmk18y9z/eval_video/8_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649fa438a50ad92ff033f4db32a883231540f22a68f0552f52a39c5668b78980 +size 1201391 diff --git a/bmk18y9z/eval_video/9_eval.mp4 b/bmk18y9z/eval_video/9_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df1dd08001e7235983f9bcbd4076fadd6815ffc7 --- /dev/null +++ b/bmk18y9z/eval_video/9_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae51d826a95728d28cae6b1b8c551126c642ebe2d2618f0dd4e5d49b45ec61d6 +size 1280834 diff --git a/bmk18y9z/tb/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 b/bmk18y9z/tb/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 new file mode 100644 index 0000000000000000000000000000000000000000..30ac63c8abed0d52a45644cc2e04628b46f9996d --- /dev/null +++ b/bmk18y9z/tb/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce0137fd8b181304ef198791ea80eb924f930763d677637b1114ae0d6d139ce +size 22784 diff --git a/bmk18y9z/wandb/debug-internal.log b/bmk18y9z/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..412522e0e105a2faf505a71babdfcb7902acf7ba --- /dev/null +++ b/bmk18y9z/wandb/debug-internal.log @@ -0,0 +1,2171 @@ +2025-01-07 22:49:26,777 INFO StreamThr :2246486 [internal.py:wandb_internal():85] W&B internal server running at pid: 2246486, started at: 2025-01-07 22:49:26.776644 +2025-01-07 22:49:26,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status +2025-01-07 22:49:26,788 INFO WriterThread:2246486 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb +2025-01-07 22:49:26,789 DEBUG SenderThread:2246486 [sender.py:send():379] send: header +2025-01-07 22:49:26,798 DEBUG SenderThread:2246486 [sender.py:send():379] send: run +2025-01-07 22:49:27,040 INFO SenderThread:2246486 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files +2025-01-07 22:49:27,040 INFO SenderThread:2246486 [sender.py:_start_run_threads():1188] run started: bmk18y9z with start time 1736308166.777832 +2025-01-07 22:49:27,054 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 22:49:27,054 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: check_version +2025-01-07 22:49:27,141 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 22:49:27,146 DEBUG HandlerThread:2246486 [system_info.py:__init__():26] System info init +2025-01-07 22:49:27,146 DEBUG HandlerThread:2246486 [system_info.py:__init__():41] System info init done +2025-01-07 22:49:27,146 INFO HandlerThread:2246486 [system_monitor.py:start():194] Starting system monitor +2025-01-07 22:49:27,146 INFO SystemMonitor:2246486 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 22:49:27,147 INFO HandlerThread:2246486 [system_monitor.py:probe():214] Collecting system info +2025-01-07 22:49:27,147 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 22:49:27,147 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started disk monitoring +2025-01-07 22:49:27,148 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 22:49:27,148 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started memory monitoring +2025-01-07 22:49:27,149 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started network monitoring +2025-01-07 22:49:27,159 DEBUG HandlerThread:2246486 [system_info.py:probe():152] Probing system +2025-01-07 22:49:27,160 DEBUG HandlerThread:2246486 [system_info.py:_probe_git():137] Probing git +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:_probe_git():145] Probing git done +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:probe():200] Probing system done +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T03:49:27.159508', 'startedAt': '2025-01-08T03:49:26.768531', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.1576250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4487.143, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 710.4931983947754}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 22:49:27,165 INFO HandlerThread:2246486 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 22:49:27,165 INFO HandlerThread:2246486 [system_monitor.py:probe():227] Publishing system info +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:_save_code():44] Saving code +2025-01-07 22:49:27,171 DEBUG HandlerThread:2246486 [system_info.py:_save_code():67] Saving code done +2025-01-07 22:49:27,171 DEBUG HandlerThread:2246486 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 22:49:27,241 DEBUG HandlerThread:2246486 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 22:49:27,241 INFO HandlerThread:2246486 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 22:49:27,245 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 22:49:27,482 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/i43982w5-wandb-metadata.json +2025-01-07 22:49:27,501 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4rgavcbr-diff.patch +2025-01-07 22:49:27,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 22:49:27,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:27,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: python_packages +2025-01-07 22:49:27,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:27,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:27,509 INFO HandlerThread:2246486 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/tb" + save: true +} +control { + relay_id: "870c06ac45b14de49a3ec250a2529b53" +} +_info { + stream_id: "bmk18y9z" +} + +2025-01-07 22:49:27,509 DEBUG HandlerThread:2246486 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 22:49:27,541 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xahuuotc-code/FISH/eval_robot.py +2025-01-07 22:49:27,603 DEBUG SenderThread:2246486 [sender.py:send():379] send: telemetry +2025-01-07 22:49:27,604 DEBUG SenderThread:2246486 [sender.py:send():379] send: tbrecord +2025-01-07 22:49:27,604 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:49:27,604 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 with policy live +2025-01-07 22:49:27,793 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4qb76c3i-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH +2025-01-07 22:49:28,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:29,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:30,041 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:30,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:31,315 DEBUG SenderThread:2246486 [sender.py:send():379] send: config +2025-01-07 22:49:31,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:32,326 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:32,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:33,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:34,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:34,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:35,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:36,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:36,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:37,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:37,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:38,044 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:38,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:39,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:40,044 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:40,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:41,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:42,045 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:42,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:42,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:42,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:42,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:43,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:44,046 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:44,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:45,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:46,046 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:46,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:47,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:48,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:48,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:49,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:50,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:51,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:52,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:53,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:54,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:54,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:55,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:56,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:57,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:57,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:57,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:58,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:59,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:59,663 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:00,050 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 22:50:00,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:01,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:02,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:03,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:04,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:05,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:05,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:06,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:07,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:08,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:09,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:10,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:11,192 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:11,517 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:12,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:50:12,158 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 with policy now +2025-01-07 22:50:12,162 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:50:12,163 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:50:12,163 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:50:12,164 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:50:12,408 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/nrhyzaq0-media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:12,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:12,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:12,517 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:13,053 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media +2025-01-07 22:50:13,343 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/2nkbwpip-media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:13,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:14,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:14,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:15,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:16,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:16,817 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:17,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:18,055 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:18,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:19,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:20,056 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:20,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:21,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:22,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:22,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:23,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:24,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:25,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:26,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:27,058 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:50:27,149 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 22:50:27,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:50:27,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:27,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:27,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:27,608 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:28,059 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:28,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:29,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:30,059 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:30,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:31,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:32,060 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:32,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:33,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:33,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:34,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:35,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:36,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:37,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:38,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:39,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:39,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:40,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:41,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:42,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:42,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:42,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:43,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:44,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:44,642 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:45,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:46,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:47,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:48,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:49,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:50,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:50,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:51,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:52,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:53,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:54,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:55,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:56,068 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:56,084 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:56,525 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:57,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:50:57,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:57,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:57,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:58,068 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:58,293 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:50:58,293 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 with policy now +2025-01-07 22:50:58,293 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:50:58,295 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:50:58,296 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:50:58,296 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:50:58,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:58,586 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/3zppzal7-media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:50:59,352 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/fhhjjeou-media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:00,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:00,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:01,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:01,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:02,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:03,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:04,070 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:04,541 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:05,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:06,071 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:06,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:06,542 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:07,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:08,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:09,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:10,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:11,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:11,329 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4of8106r-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:11,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:11,842 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:11,909 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:11,909 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:11,909 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:12,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:12,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:12,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:12,509 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:12,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:13,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:14,074 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:14,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:15,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:16,075 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:16,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:16,940 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:17,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:18,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:19,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:20,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:21,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:21,941 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:22,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:23,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:24,077 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:24,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:25,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:26,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:26,946 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:27,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:51:27,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:27,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:27,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:28,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:29,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:30,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:31,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:32,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:32,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:33,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:34,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:35,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:36,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:37,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:37,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:38,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:39,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:40,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:41,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:42,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:42,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:42,511 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:42,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:42,665 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:42,809 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:51:42,809 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:51:42,809 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 with policy now +2025-01-07 22:51:42,810 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:42,810 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:42,810 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:43,068 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/09he7tcy-media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:51:43,344 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/54vbj6wu-media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:44,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:44,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:45,540 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:46,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:47,544 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:48,086 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:48,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:48,545 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:49,544 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:50,086 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:50,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:51,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:52,556 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:53,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:54,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:54,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:55,088 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:55,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:55,741 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:55,741 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:55,741 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:56,088 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:56,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:56,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:56,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:57,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:51:57,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:57,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:57,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:58,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:58,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:59,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:00,090 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:00,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:00,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:01,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:02,090 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:02,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:03,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:04,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:05,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:05,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:06,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:07,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:08,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:09,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:10,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:11,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:11,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:12,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:12,511 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:12,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:13,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:14,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:15,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:16,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:16,624 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:17,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:18,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:19,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:20,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:21,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:22,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:22,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:23,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:24,098 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:24,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:25,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:26,557 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:27,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:52:27,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:27,516 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:27,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:52:27,558 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:52:27,600 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 with policy now +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:52:27,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:27,601 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:52:27,862 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/dy3n98w3-media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,100 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:52:28,376 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4tof1kf1-media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:29,563 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:30,100 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:30,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:31,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:32,101 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:32,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:33,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:33,565 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:34,567 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:35,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:36,102 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:36,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:37,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:38,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:39,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:39,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:40,103 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:52:40,364 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/ygnz6olk-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:52:40,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:40,799 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:52:40,799 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:52:40,800 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:52:41,104 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:52:41,570 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:42,104 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:42,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:42,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:42,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:43,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:44,581 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:44,945 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:45,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:46,106 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:46,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:47,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:48,106 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:48,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:49,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:50,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:50,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:51,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:52,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:53,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:54,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:55,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:55,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:56,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:57,152 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:52:57,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:57,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:57,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:58,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:59,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:00,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:00,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:01,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:02,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:03,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:04,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:05,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:06,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:06,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:07,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:08,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:09,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:10,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:11,575 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:12,209 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:12,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:12,518 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:12,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:13,580 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:53:13,580 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 with policy now +2025-01-07 22:53:13,580 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:13,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:53:13,583 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:13,583 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:13,584 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:13,831 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/pcomtarx-media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:53:14,405 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/swjopcct-media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:15,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:16,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:16,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:17,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:17,585 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:18,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:18,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:19,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:20,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:21,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:22,118 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:22,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:22,596 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:23,596 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:24,601 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:24,850 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:24,851 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:24,852 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:25,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:53:25,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:25,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:26,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:26,593 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:27,152 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:53:27,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:27,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:27,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:27,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:28,120 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:28,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:29,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:30,121 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:30,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:31,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:32,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:33,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:33,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:34,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:35,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:36,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:37,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:38,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:38,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:39,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:40,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:41,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:42,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:42,515 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:42,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:43,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:43,608 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:44,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:45,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:46,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:47,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:48,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:49,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:49,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:50,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:51,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:52,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:53,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:54,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:54,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:55,614 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:56,615 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:57,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:53:57,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:53:57,154 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 with policy now +2025-01-07 22:53:57,162 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:53:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:57,162 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:57,163 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:57,417 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/qm8tagd0-media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:57,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:57,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:57,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:58,130 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:53:58,431 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/u3i0jw7h-media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:58,618 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:59,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:59,624 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:00,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:00,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:01,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:02,132 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:02,626 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:03,619 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:04,133 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:04,632 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:05,449 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:05,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:06,133 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:06,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:07,622 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:08,649 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:08,907 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:08,907 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:08,908 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:09,134 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:09,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:09,421 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/6aw8p6vy-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:09,626 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:10,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:10,632 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:10,745 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:11,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:12,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:12,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:12,518 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:12,627 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:13,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:14,136 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:14,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:15,627 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:16,137 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:16,414 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:16,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:17,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:18,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:19,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:20,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:21,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:21,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:22,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:23,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:24,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:25,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:26,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:26,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:27,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:54:27,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:27,516 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:27,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:28,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:29,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:30,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:31,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:31,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:32,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:33,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:34,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:35,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:36,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:37,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:37,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:38,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:39,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:40,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:41,640 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:42,147 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:42,280 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:54:42,280 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 with policy now +2025-01-07 22:54:42,281 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:54:42,282 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:42,283 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:42,284 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:42,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:42,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:42,546 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/e4pyyz5y-media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:42,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:42,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:54:43,454 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/q99puyrq-media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:43,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:44,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:44,640 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:45,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:46,651 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:47,655 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:48,150 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:48,540 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:48,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:49,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:50,150 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:50,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:51,653 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:52,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:53,543 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:53,657 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:54,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:54,966 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:54,966 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:54,967 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:55,152 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:55,152 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:55,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:56,153 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:56,653 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:57,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:54:57,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:57,535 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:57,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:58,153 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:58,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:58,669 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:59,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:00,154 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:00,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:01,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:02,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:03,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:04,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:04,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:05,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:06,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:07,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:08,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:09,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:09,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:10,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:11,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:12,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:12,522 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:12,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:13,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:14,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:14,658 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:15,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:16,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:17,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:18,649 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:19,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:20,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:20,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:21,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:22,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:23,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:24,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:25,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:25,657 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:26,666 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:27,021 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:55:27,021 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 with policy now +2025-01-07 22:55:27,031 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:55:27,031 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:55:27,031 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:55:27,032 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:55:27,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:55:27,275 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/u89v0n8x-media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:27,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:27,548 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7d1rxucs-media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,686 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:28,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:28,664 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:29,669 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:30,545 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:30,665 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:31,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:32,166 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:32,668 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:33,662 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:34,166 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:34,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:35,674 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:36,477 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:36,674 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:37,670 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:38,666 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:39,018 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:55:39,018 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:55:39,018 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:55:39,168 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:55:39,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:55:39,421 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7vbnlg3i-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:55:39,693 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:40,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:40,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:41,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:41,693 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:42,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:42,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:42,532 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:42,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:43,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:44,170 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:44,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:45,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:46,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:46,867 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:47,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:48,172 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:48,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:49,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:50,173 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:50,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:51,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:52,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:52,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:53,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:54,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:55,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:56,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:57,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:55:57,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:57,523 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:57,634 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:57,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:58,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:59,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:00,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:01,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:02,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:03,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:03,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:04,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:05,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:06,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:07,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:08,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:08,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:09,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:10,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:11,317 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:56:11,317 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 with policy now +2025-01-07 22:56:11,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:56:11,320 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:56:11,320 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:56:11,320 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:56:11,565 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/3c60a9nl-media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:11,694 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:12,182 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:56:12,469 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xpqoq56m-media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:12,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:12,539 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:12,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:13,695 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:13,695 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:14,182 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:14,717 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:15,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:16,183 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:16,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:17,699 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:18,726 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:19,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:19,715 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:20,184 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:20,730 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:21,725 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:22,729 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:23,734 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:24,072 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:56:24,072 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:56:24,073 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:56:24,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:56:24,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:56:24,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:25,080 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:25,745 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:26,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:26,750 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:27,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:56:27,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:27,534 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:27,734 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:28,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:29,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:30,188 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:30,735 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:31,017 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:31,755 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:32,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:33,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:34,190 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:34,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:35,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:36,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:36,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:37,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:38,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:39,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:40,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:41,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:41,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:42,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:42,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:42,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:43,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:44,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:45,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:46,611 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:46,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:47,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:48,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:49,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:50,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:51,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:51,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:52,197 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:52,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:53,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:54,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:55,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:56,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:57,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:56:57,156 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:57,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:57,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:57,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:58,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:59,754 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:00,008 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:57:00,008 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 with policy now +2025-01-07 22:57:00,012 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:57:00,013 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:00,013 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:00,013 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:57:00,286 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/soubxlgx-media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,554 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gmbe9tvy-media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,760 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:01,764 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:02,201 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:02,561 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:02,758 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:03,770 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:04,766 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:05,767 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:06,202 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:06,762 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:07,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:07,775 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:08,203 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:08,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:09,781 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:10,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:11,772 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:12,546 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:12,546 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:12,694 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:12,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:13,793 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:14,785 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:15,798 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:16,136 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:16,137 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:16,137 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:57:16,457 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mlbpxsy1-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:57:16,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:17,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:18,208 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:18,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:18,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:19,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:20,209 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:20,801 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:21,805 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:22,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:23,664 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:23,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:24,211 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:24,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:25,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:26,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:27,156 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:57:27,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:27,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:27,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:28,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:29,588 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:29,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:30,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:31,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:32,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:33,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:34,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:35,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:35,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:36,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:37,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:38,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:39,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:40,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:40,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:41,798 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:42,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:42,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:42,801 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:43,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:44,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:45,659 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:45,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:46,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:47,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:48,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:49,838 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:50,221 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:50,515 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:57:50,515 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 with policy now +2025-01-07 22:57:50,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:57:50,525 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:50,525 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:50,526 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:50,772 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/lyi72qj3-media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:50,816 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:51,221 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:51,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:51,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:57:51,511 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/2u0zbof9-media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:51,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:51,823 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:52,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:52,830 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:53,815 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:54,817 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:55,836 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:56,223 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:56,830 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:57,118 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:57,156 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:57:57,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:57,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:57,831 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:58,224 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:58,847 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:59,850 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:00,843 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:01,846 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:02,192 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:02,193 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:02,193 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:02,193 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:02,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:02,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:02,845 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:03,843 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:04,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:04,857 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:05,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:06,851 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:07,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:07,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:08,228 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:08,874 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:09,868 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:10,877 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:11,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:12,229 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:12,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:12,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:12,642 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:12,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:13,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:14,865 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:15,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:16,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:17,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:18,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:18,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:19,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:20,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:21,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:22,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:23,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:23,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:24,854 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:25,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:26,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:27,157 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:58:27,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:27,532 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:27,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:28,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:28,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:29,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:30,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:31,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:32,861 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:33,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:34,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:34,869 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:35,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:36,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:37,870 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:38,877 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:39,553 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:58:39,553 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 with policy now +2025-01-07 22:58:39,556 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:58:39,565 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:39,566 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:39,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:39,566 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:39,842 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/0jqf5dag-media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:39,882 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:58:40,565 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gisqoycc-media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:40,875 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:41,906 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:42,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:42,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:42,564 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:42,909 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:43,915 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:44,242 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:44,903 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:45,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:45,948 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:46,904 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:47,904 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:48,244 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:48,911 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:49,910 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:50,579 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:50,927 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:51,925 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:52,930 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:53,930 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:54,937 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:55,939 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:56,217 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:56,247 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:56,254 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:56,254 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:56,255 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:56,957 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:57,157 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:58:57,247 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:57,248 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:57,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:57,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:57,940 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:58,248 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:58,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:59,966 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:00,985 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:01,397 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:01,966 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:02,249 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:02,962 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:03,962 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:04,970 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:05,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:06,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:07,230 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:07,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:08,252 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:08,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:09,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:10,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:11,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:12,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:12,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:12,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:12,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:13,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:14,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:15,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:16,255 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:16,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:17,952 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:18,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:18,957 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:19,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:20,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:21,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:22,257 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:22,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:23,578 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:23,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:24,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:25,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:26,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:27,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:59:27,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:27,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:27,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:28,641 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:28,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:29,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:30,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:31,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:32,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:33,828 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:33,977 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:34,262 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:34,664 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:59:34,664 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 with policy now +2025-01-07 22:59:34,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:59:34,681 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:59:34,681 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:59:34,682 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:59:34,918 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/eh83y9ua-media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:34,965 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:35,262 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:59:35,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:35,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:59:35,557 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/z1e4jqc8-media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:35,979 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:36,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:36,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:37,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:38,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:39,309 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:39,995 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:40,265 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:40,997 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:41,980 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:42,266 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:42,561 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:42,561 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:42,983 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:43,996 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:44,399 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:45,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:45,267 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:59:45,330 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:59:45,330 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:59:45,330 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:59:45,573 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xrrocyx2-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:59:46,018 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:46,268 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:46,268 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:59:47,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:48,022 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:49,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:49,899 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:50,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:50,270 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:51,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:52,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:53,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:54,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:55,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:55,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:56,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:57,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:57,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:59:57,538 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:57,538 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:58,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:59,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:00,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:00,593 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:01,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:02,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:03,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:04,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:05,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:06,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:06,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:07,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:08,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:09,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:10,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:11,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:12,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:12,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:12,539 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:12,687 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:13,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:14,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:15,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:16,010 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:16,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:17,012 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:17,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:00:17,150 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 with policy now +2025-01-07 23:00:17,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:00:17,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:00:17,153 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:00:17,153 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:00:17,409 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/tqbwf0se-media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:17,685 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/1kylfdea-media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:18,028 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:18,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:18,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:19,016 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:20,018 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:21,021 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:22,020 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:22,283 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:23,037 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:23,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:24,014 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:24,284 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:25,019 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:26,030 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:27,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:27,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:00:27,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:27,573 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:28,034 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:28,718 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:29,022 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:30,032 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:30,286 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:00:30,386 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:00:30,386 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:00:30,387 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:00:31,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:31,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:00:31,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:00:32,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:32,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:33,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:34,030 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:34,305 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:35,040 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:36,048 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:36,289 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:37,055 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:38,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:39,055 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:39,625 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:40,051 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:41,064 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:42,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:42,291 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:42,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:42,547 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:43,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:44,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:44,697 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:45,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:46,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:47,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:48,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:49,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:50,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:50,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:51,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:52,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:53,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:54,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:55,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:56,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:56,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:57,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:57,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:00:57,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:57,547 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:58,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:59,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:00,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:01,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:01,672 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:02,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:03,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:04,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:05,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:06,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:07,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:07,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:08,081 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:09,086 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:09,651 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:01:09,651 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 with policy now +2025-01-07 23:01:09,656 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:01:09,657 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:01:09,657 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:01:09,658 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:01:09,902 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/pued8wpr-media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:10,067 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:01:10,585 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4zdrllnr-media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:11,086 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:12,094 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:12,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:12,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:12,574 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:12,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:13,085 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:14,071 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:14,305 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:15,075 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:16,072 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:17,081 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:18,093 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:18,307 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:18,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:19,078 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:20,090 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:21,092 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:22,097 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:23,095 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:23,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:24,093 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:25,114 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:26,104 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:27,123 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:27,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:01:27,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:27,582 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:28,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:28,645 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:29,129 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:30,128 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:30,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:01:30,459 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:01:30,459 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:01:30,459 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:01:31,121 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:31,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:31,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:01:32,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:33,170 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:33,313 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:33,857 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:34,126 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:35,131 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:35,314 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:36,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:37,165 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:38,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:39,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:39,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:40,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:41,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:41,317 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:42,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:42,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:42,551 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:43,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:43,318 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:44,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:44,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:45,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:46,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:47,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:47,319 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:48,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:49,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:49,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:50,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:51,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:52,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:53,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:54,153 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:55,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:55,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:56,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:57,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:57,160 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:01:57,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:57,551 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:58,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:59,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:00,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:00,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:01,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:02,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:03,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:04,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:05,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:06,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:06,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:07,174 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:08,183 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:08,253 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:02:08,253 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:02:08,254 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 with policy now +2025-01-07 23:02:08,254 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:02:08,254 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:02:08,259 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:02:08,497 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/be9yfc7s-media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:08,768 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mzfux2k8-media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:09,206 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:09,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:10,186 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:10,329 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:11,204 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:11,329 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:11,909 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:12,213 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:12,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:12,568 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:13,196 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:13,330 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:14,194 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:15,185 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:16,214 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:17,077 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:17,210 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:17,332 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:18,206 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:19,210 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:20,221 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:21,209 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:22,222 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:22,668 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:23,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:24,220 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:24,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:02:24,520 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:02:24,521 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:02:24,521 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:02:24,592 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/k386vs5c-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:02:25,216 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:25,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:25,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:02:26,226 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:27,160 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:02:27,237 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:27,336 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:27,579 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:27,580 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:27,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:28,232 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:29,242 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:30,242 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:31,241 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:31,338 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:32,251 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:33,256 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:33,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:34,252 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:35,257 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:36,283 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:37,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:37,340 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:38,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:39,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:39,341 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:39,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:40,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:41,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:42,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:42,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:42,568 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:43,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:44,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:44,715 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:45,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:46,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:47,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:48,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:49,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:50,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:50,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:51,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:52,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:53,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:53,347 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:54,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:55,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:55,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:56,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:57,161 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:02:57,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:57,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:57,569 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:58,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:59,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:00,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:01,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:01,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:02,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:03,285 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:04,273 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:04,593 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:03:04,594 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 with policy now +2025-01-07 23:03:04,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:03:04,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:04,600 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:04,607 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:04,864 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/zsgq8r8n-media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:05,260 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:03:05,642 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/bntajr09-media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:06,275 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:06,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:06,728 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:07,286 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:07,353 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:08,291 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:09,298 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:10,335 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:11,289 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:11,354 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:12,299 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:12,541 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:12,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:12,572 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:13,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:13,355 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:14,318 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:15,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:16,314 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:17,297 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:17,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:18,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:18,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:03:18,586 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:18,586 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:18,586 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:19,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:19,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:19,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:20,297 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:21,305 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:21,359 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:22,296 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:23,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:23,703 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:24,318 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:25,304 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:25,360 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:26,312 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:27,161 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:03:27,317 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:27,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:27,595 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:28,327 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:28,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:29,344 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:30,355 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:31,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:31,363 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:32,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:33,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:33,364 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:34,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:34,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:35,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:36,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:37,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:38,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:39,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:40,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:40,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:41,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:42,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:42,587 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:42,588 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:43,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:44,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:45,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:45,711 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:46,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:47,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:48,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:49,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:50,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:51,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:51,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:52,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:53,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:54,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:55,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:56,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:56,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:03:57,325 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:57,605 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:57,605 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:58,348 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:58,531 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:03:58,531 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 with policy now +2025-01-07 23:03:58,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:03:58,534 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:58,534 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:58,535 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:58,792 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7shd7hdg-media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:03:59,361 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:59,380 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:59,380 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:59,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:03:59,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:03:59,678 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4om8pg02-media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:04:00,335 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:00,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:01,341 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:01,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:02,254 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:02,371 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:03,353 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:04,358 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:05,383 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:05,385 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:06,378 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:07,367 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:07,384 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:07,581 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:08,359 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:09,381 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:10,358 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:11,380 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:12,356 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:12,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:04:12,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:12,599 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:12,712 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:04:12,712 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:04:12,712 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:12,713 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:04:13,375 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:13,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:13,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:04:14,407 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:15,387 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:15,436 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:16,385 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:17,389 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:17,766 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:18,392 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:19,389 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:19,390 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:20,395 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:21,396 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:22,392 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:23,413 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:23,760 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:24,409 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:25,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:26,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:27,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:04:27,392 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:27,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:27,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:27,589 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:28,404 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:28,888 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:29,405 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:30,411 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:31,394 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:31,415 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:32,418 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:33,395 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:33,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:34,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:34,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:35,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:36,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:37,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:38,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:39,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:40,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:40,725 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:41,398 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:41,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:42,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:42,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:42,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:43,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:44,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:45,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:45,913 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:46,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:47,401 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:47,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:48,424 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:49,424 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:50,431 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:51,431 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:51,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:52,440 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:53,403 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:53,465 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:53,648 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:04:53,649 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 with policy now +2025-01-07 23:04:53,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:04:53,652 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:04:53,653 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:04:53,653 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:04:53,895 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xiilgsy3-media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:04:54,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:54,775 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/b8s9beoi-media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:55,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:55,446 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:56,405 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:56,472 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:04:57,163 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:57,446 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:57,605 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:57,605 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:58,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:59,406 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:59,473 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:00,475 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:01,477 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:02,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:02,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:03,408 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:03,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:04,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:04,482 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:04,715 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:04,715 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:04,716 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:05,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:05,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:05,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:06,499 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:07,410 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:07,474 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:07,802 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:08,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:09,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:10,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:11,412 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:11,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:12,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:12,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:12,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:13,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:13,776 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:14,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:15,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:16,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:17,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:18,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:19,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:19,776 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:20,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:21,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:22,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:23,417 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:23,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:24,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:25,418 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:25,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:25,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:26,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:27,163 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:05:27,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:27,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:27,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:28,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:29,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:30,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:30,779 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:31,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:32,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:33,470 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:34,470 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:35,488 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:36,328 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:36,475 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:37,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:37,490 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:38,143 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:38,143 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 with policy now +2025-01-07 23:05:38,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:38,147 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:38,147 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:38,147 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:38,403 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/i4axdrst-media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:38,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:05:38,478 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:38,689 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/fvf77b99-media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:39,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:39,484 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:40,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:40,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:41,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:41,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:42,491 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:42,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:42,614 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:43,426 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:43,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:44,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:45,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:46,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:46,981 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:47,428 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:47,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:48,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:49,429 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:49,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:49,689 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/567ghusu-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:49,770 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:49,771 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:49,771 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:50,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:50,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:50,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:05:50,562 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:05:50,878 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:51,067 INFO wandb-upload_1:2246486 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpanrkz_f3 +2025-01-07 23:05:51,350 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:05:51,359 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:51,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:51,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval +2025-01-07 23:05:51,482 INFO SenderThread:2246486 [sender.py:send_request_log_artifact():1525] logged artifact run-bmk18y9z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjYxMzY1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkxNTcwMA==', 'latestArtifact': None}} +2025-01-07 23:05:51,482 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:51,482 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json with policy now +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,484 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send():379] send: telemetry +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,485 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,485 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,485 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,486 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,486 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,486 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,487 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,487 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,487 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,487 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,488 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,488 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,488 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,489 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,489 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,489 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,489 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,490 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,490 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,490 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,490 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,491 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:05:51,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:51,664 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/uxopy9e7-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:51,717 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mphxn0r0-media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:51,891 INFO wandb-upload_0:2246486 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0wk230bn +2025-01-07 23:05:51,930 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/cw8ac4bk-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:52,288 INFO SenderThread:2246486 [sender.py:send_request_log_artifact():1525] logged artifact run-bmk18y9z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjYxMzY3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkxNTcwOQ==', 'latestArtifact': None}} +2025-01-07 23:05:52,288 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:52,288 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json with policy now +2025-01-07 23:05:52,289 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:52,289 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:52,289 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:52,289 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:52,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:52,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:52,544 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gyqkjwnx-media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:53,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:53,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:54,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:55,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:56,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:57,170 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:05:57,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:57,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:57,597 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:57,753 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:58,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:59,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:00,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:01,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:02,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:02,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:03,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:04,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:05,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:06,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:07,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:08,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:08,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:09,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:10,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:11,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:12,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:12,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:06:12,598 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:06:13,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:13,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:14,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:15,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:16,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:17,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:18,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:19,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:19,787 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:20,445 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 23:06:20,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:21,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:21,581 DEBUG SenderThread:2246486 [sender.py:send():379] send: exit +2025-01-07 23:06:21,581 INFO SenderThread:2246486 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:send_exit():588] handling runtime: 1014 +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:send_exit():594] send defer +2025-01-07 23:06:21,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,582 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 23:06:21,582 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 1 +2025-01-07 23:06:21,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 23:06:21,583 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 2 +2025-01-07 23:06:21,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 23:06:21,584 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 23:06:21,584 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 23:06:21,635 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 23:06:21,636 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 23:06:21,636 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined network monitor +2025-01-07 23:06:21,636 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,636 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 23:06:21,636 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 3 +2025-01-07 23:06:21,636 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:06:21,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,637 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 23:06:21,637 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,637 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 23:06:21,637 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 4 +2025-01-07 23:06:21,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,637 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 23:06:22,446 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:23,447 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 23:06:27,781 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,781 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 23:06:27,781 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 5 +2025-01-07 23:06:27,781 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:27,782 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:06:27,782 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:06:27,782 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,783 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 23:06:27,783 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:27,784 DEBUG SenderThread:2246486 [sender.py:send():379] send: summary +2025-01-07 23:06:27,787 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:27,787 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,787 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 6 +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,788 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 23:06:27,788 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 7 +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,788 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 23:06:27,788 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 23:06:28,449 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:28,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:30,213 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 8 +2025-01-07 23:06:30,214 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:30,214 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,214 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 23:06:30,214 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,214 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 23:06:30,214 INFO SenderThread:2246486 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 23:06:30,217 INFO SenderThread:2246486 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 23:06:30,219 INFO SenderThread:2246486 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 23:06:30,227 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 9 +2025-01-07 23:06:30,228 DEBUG SenderThread:2246486 [sender.py:send():379] send: artifact +2025-01-07 23:06:30,230 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,231 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 23:06:30,430 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml config.yaml +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json wandb-summary.json +2025-01-07 23:06:30,451 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt requirements.txt +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json wandb-metadata.json +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log output.log +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch diff.patch +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 23:06:30,458 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 10 +2025-01-07 23:06:30,461 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,461 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 23:06:30,461 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,461 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 23:06:30,461 INFO SenderThread:2246486 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:06:30,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:30,584 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:30,630 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:30,647 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 23:06:30,720 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt +2025-01-07 23:06:30,724 INFO wandb-upload_4:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:30,733 INFO wandb-upload_3:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/9hmlgza3-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:06:30,934 INFO Thread-11 (_thread_body):2246486 [sender.py:transition_state():614] send defer: 11 +2025-01-07 23:06:30,934 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,934 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 23:06:30,934 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,935 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 23:06:30,935 INFO SenderThread:2246486 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 12 +2025-01-07 23:06:30,938 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,938 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 23:06:30,938 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [file_stream.py:finish():601] file stream finish called +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [file_stream.py:finish():605] file stream finish is done +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 13 +2025-01-07 23:06:31,115 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:31,115 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 23:06:31,115 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 14 +2025-01-07 23:06:31,115 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:31,115 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send():379] send: final +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send():379] send: footer +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:31,116 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 23:06:31,116 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 23:06:31,118 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:31,118 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: server_info +2025-01-07 23:06:31,182 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:31,182 INFO MainThread:2246486 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 23:06:31,183 INFO MainThread:2246486 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 23:06:31,183 INFO MainThread:2246486 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 23:06:31,184 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 23:06:31,184 INFO HandlerThread:2246486 [handler.py:finish():882] shutting down handler +2025-01-07 23:06:32,118 INFO WriterThread:2246486 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [sender.py:finish():1615] shutting down sender +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [file_pusher.py:join():175] waiting for file pusher diff --git a/bmk18y9z/wandb/debug.log b/bmk18y9z/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70464a8cb3f0d2b819fefb05459df734a59e8e6d --- /dev/null +++ b/bmk18y9z/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 22:49:26,770 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 22:49:26,770 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Configure stats pid to 2246319 +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/settings +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug.log +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug-internal.log +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():569] calling init triggers +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z'} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():619] starting backend +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():623] setting up manager +2025-01-07 22:49:26,775 INFO MainThread:2246319 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 22:49:26,777 INFO MainThread:2246319 [wandb_init.py:init():631] backend started and connected +2025-01-07 22:49:26,788 INFO MainThread:2246319 [wandb_init.py:init():720] updated telemetry +2025-01-07 22:49:26,795 INFO MainThread:2246319 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 22:49:27,053 INFO MainThread:2246319 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 22:49:27,093 INFO MainThread:2246319 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 22:49:27,093 INFO MainThread:2246319 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 22:49:27,506 INFO MainThread:2246319 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 22:49:27,506 INFO MainThread:2246319 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 22:49:27,507 INFO MainThread:2246319 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 22:49:27,507 INFO MainThread:2246319 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 22:49:27,508 INFO MainThread:2246319 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 22:49:27,509 INFO MainThread:2246319 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/tb, True +2025-01-07 22:49:31,314 INFO MainThread:2246319 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 23:06:32,184 WARNING MsgRouterThr:2246319 [router.py:message_loop():77] message_loop has been closed diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55921f0924911e2ff3743ec58eb8a9b6b0d6a4d8 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1002_0' +wandb_notes: + desc: null + value: 1002_0_req_1365_0 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 20 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736308166 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..3f33d68d5f61ef69d5b4133841e12781c858f071 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch @@ -0,0 +1,30 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..0b377ae 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 5 ++num_eval: 20 + + debug_timestamps: False + open_loop: False +@@ -133,14 +133,14 @@ load_checkpoint: ${agent.load_checkpoint} + + # all books, 10/20 demos per book + # RGBD+mask+act history +-# wandb_run_id: '1002_0' ++wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 + # wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json new file mode 100644 index 0000000000000000000000000000000000000000..b10d2043a20baa9d07c1baf6c0802d5f9b401012 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.3, 0.1003983021581674, 0.558511674392442, 0.1215232685965274, 0.502513885497544, 0.16696532077297443, 0.48316777497866625, 0.17897007993323794, 0.4645809233259468]]} \ No newline at end of file diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d93d0d4cff16be73c844ac2cedca8c1e93091843 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[47, 145, 0.32413793103448274, 0.24027697747179777, 0.41904157401750397, 0.25318085779715693, 0.40072802456218604, 0.26618888606336943, 0.3878559880182919, 0.2730443329072916, 0.3766404036679225]]} \ No newline at end of file diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..076d3a037c337c0b15364b43d24198c548f6c98e --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a7acf7610c7a7f0d62e8d433ce287a8947ce47c8f1195c93b999c25ecce79b3 +size 1138909 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9bb0923bdbe831fdca211a9dbdaf558bd776d297 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c5570816ad9da40a43e521c2692fbf3c896a7868112e102639b1f803807b568 +size 1278356 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..40216455fa2cb1ff1e6f371b61316088d0453eee --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a606d9c48c91655df31c54ec48b57edcb5ae66c9e0e72e08d0040360683cbca7 +size 1399219 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..57babdb07ee97e0ebe6c5ec0a987786bd79035f3 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a187eeb09299696aa2a1651b0c5885f8d4888b203fafb6872a4451f7c71ffef +size 1427340 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8eca45e01bc4ce3cca52ee13d4aa08b6e83ba697 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9efca5d983ee670cf985545f839f820290fa9648c2efe9805271496b3e2afe5c +size 1205126 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e0c7a896759ae2d671156dabd4247f773434c5e2 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffa92cd8767b9ad59d685b39c030e4373019479e38dcd00037ea91346b07353d +size 1373307 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..51ee392298feb5331cadf997acd64a97ea5d487a --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af536578f4476ce23195e3698f30ff9f31c2321e1d8a9efb7a5d698910722d35 +size 1358907 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b38ebf7122b784a810a6dffd4375edd8423b261e --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db31934236335d9e75e4f0c34cec84e72d542118c7c4ee6ce4f5de77075484b3 +size 1444676 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67fbebd68e970c314e3990990dcb4c40e6853146 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0293b276af8b7ee82ad1af37fc416a012c67d2159e42b28d61210c141cd56c7 +size 1445556 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b19802a6ade1903a09c0d86e7e87bd75d0e3c929 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:205c297dae90682b6f426123d9088c72777d0add28b23f8d2ff0181423bf5020 +size 1459635 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a091605756e16e95fb53840d565df23e5b73573 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b3cf56f899624bcd38b38ef615d544991af40aab002b25fe8e00be0095a8ca +size 1191669 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a117341396ed7587e221304261e7cef9e08520f --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6105fa800baa66794d18fe8647635294b8fd467674d9a85bfd6647039cf1293d +size 1193600 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..826803eff02e64384ed8b2525da0dfecc90d3249 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92292561966baf2ee3899d136adc87f373197924f837a6a3dddb13dd7dfc730 +size 1143421 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2474d7bc0b6ce608db49f3ba5cfb4bd40e89be80 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4978aa6435b32ecf67343f7f22cb866f6a82d85d5d1892976fb019fffb095a08 +size 1200135 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dce9f5ab09fe94cbe396bd0ff885e6be04a18b2e --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecf6c04acd951daf42e0863e775e650eb62d0fba1c2c91368af151a2718d6a42 +size 1247071 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5aaa39f3a82a24e23726ebdf55c992fcbad75e5a --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c452dd651b0df318063859d846c3dd0082130d16fe37e52f60da91cffdecb788 +size 1198593 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dc68d7d0db83f560b28dce118b8ea16dbc0c2f47 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a40bb3e24616a11f81cf8f8cb531134b91ca09b4d31f5c66d26081f8a597697a +size 1243138 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa48398d2adedf7528a12830d290d9f235981eb7 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1822051540a8ae17e31542973fd9483b3e04543b61fa35aa2e112db4e7930691 +size 1216136 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1769b65b894045d3c4a43de78f4468d40f69db77 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:649fa438a50ad92ff033f4db32a883231540f22a68f0552f52a39c5668b78980 +size 1201391 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df1dd08001e7235983f9bcbd4076fadd6815ffc7 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae51d826a95728d28cae6b1b8c551126c642ebe2d2618f0dd4e5d49b45ec61d6 +size 1280834 diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..fde62b96e442a5b38cf934df2d191c0805d04daa --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log @@ -0,0 +1,1104 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736308171.315333]: resetting environment +[INFO] [1736308171.321522]: cleared current plan +[INFO] [1736308171.322673]: moving to home +[INFO] [1736308172.325301]: reached home +[INFO] [1736308172.327230]: reset action history +[INFO] [1736308175.034437]: environment reset +Starting episode 0 +[INFO] [1736308175.036847]: resetting environment +[INFO] [1736308175.039753]: cleared current plan +[INFO] [1736308175.040577]: moving to home +[INFO] [1736308176.043001]: reached home +[INFO] [1736308176.044589]: reset action history +[INFO] [1736308178.753785]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736308180.402866]: resetting environment +[INFO] [1736308180.407567]: cleared current plan +[INFO] [1736308180.408406]: moving to home +[INFO] [1736308181.410767]: reached home +[INFO] [1736308181.412044]: reset action history +[INFO] [1736308184.120682]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_0_2025-01-07-22-49-44.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:46035/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2246589] +process[print_text-2]: started with pid [2246613] +process[rosbag_record-3]: started with pid [2246614] +started bagging! +[INFO] [1736308211.191998]: Storing episode... +[pub_text-1] killing on exit +[print_text-2] killing on exit[rosbag_record-3] killing on exit +[INFO] [1736308211.968372]: Stored episode 1. +[INFO] [1736308211.968630]: Saving video... +[INFO] [1736308212.147079]: Video saved! +Starting episode 1 +[INFO] [1736308212.165730]: resetting environment +[INFO] [1736308212.169989]: cleared current plan +[INFO] [1736308212.170203]: moving to home +[INFO] [1736308215.772500]: reached home +[INFO] [1736308215.773825]: reset action history +[INFO] [1736308218.486504]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308226.376851]: resetting environment +[INFO] [1736308226.381339]: cleared current plan +[INFO] [1736308226.381518]: moving to home +[INFO] [1736308227.382327]: reached home +[INFO] [1736308227.382725]: reset action history +[INFO] [1736308230.089348]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_1_2025-01-07-22-50-30.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:33767/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2246725] +process[print_text-5]: started with pid [2246726] +process[rosbag_record-6]: started with pid [2246750] +started bagging! +For topic gripper_width: timestamp difference is 102483571 for nearest: 1736308254872535289 - target: 1736308254770051718 at idx 249 +For topic gripper_width: timestamp difference is 102649241 for nearest: 1736308255072534829 - target: 1736308254969885588 at idx 249 +[INFO] [1736308257.051676]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736308257.821417]: Stored episode 2. +[INFO] [1736308257.821691]: Saving video... +[INFO] [1736308258.265940]: Video saved! +Starting episode 2 +[INFO] [1736308258.295514]: resetting environment +[INFO] [1736308258.306451]: cleared current plan +[INFO] [1736308258.306697]: moving to home +[INFO] [1736308261.709626]: reached home +[INFO] [1736308261.714054]: reset action history +[INFO] [1736308264.492710]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308270.830471]: resetting environment +[INFO] [1736308270.835042]: cleared current plan +[INFO] [1736308270.837435]: moving to home +[INFO] [1736308271.840557]: reached home +[INFO] [1736308271.841885]: reset action history +[INFO] [1736308274.547504]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_2_2025-01-07-22-51-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:44937/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2246883] +process[print_text-8]: started with pid [2246884] +process[rosbag_record-9]: started with pid [2246885] +started bagging! +For topic gripper_width: timestamp difference is -109046286 for nearest: 1736308281705868179 - target: 1736308281814914465 at idx 106 +For topic gripper_width: timestamp difference is -110353052 for nearest: 1736308282706901968 - target: 1736308282817255020 at idx 118 +For topic gripper_width: timestamp difference is 106414434 for nearest: 1736308301072537061 - target: 1736308300966122627 at idx 249 +[INFO] [1736308301.680207]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +For topic gripper_width: timestamp difference is -193047915 for nearest: 1736308301405892219 - target: 1736308301598940134 at idx 249 +[INFO] [1736308302.425656]: Stored episode 3. +[INFO] [1736308302.426079]: Saving video... +[INFO] [1736308302.767089]: Video saved! +Starting episode 3 +[INFO] [1736308302.800471]: resetting environment +[INFO] [1736308302.811124]: cleared current plan +[INFO] [1736308302.816784]: moving to home +[INFO] [1736308306.221337]: reached home +[INFO] [1736308306.223618]: reset action history +[INFO] [1736308309.176860]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308315.096220]: resetting environment +[INFO] [1736308315.105833]: cleared current plan +[INFO] [1736308315.113301]: moving to home +[INFO] [1736308316.115195]: reached home +[INFO] [1736308316.120856]: reset action history +[INFO] [1736308318.888715]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_3_2025-01-07-22-51-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:38475/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2247019] +process[print_text-11]: started with pid [2247020] +process[rosbag_record-12]: started with pid [2247021] +started bagging! +For topic gripper_width: timestamp difference is -259819766 for nearest: 1736308342705868463 - target: 1736308342965688229 at idx 249 +[INFO] [1736308346.033635]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736308346.812782]: Stored episode 4. +[INFO] [1736308346.814612]: Saving video... +[WARN] [1736308347.413649]: Plan exhausted +[WARN] [1736308347.440834]: Plan exhausted +[WARN] [1736308347.480101]: Plan exhausted +[WARN] [1736308347.522455]: Plan exhausted +[INFO] [1736308347.472345]: Video saved! +Starting episode 4 +[INFO] [1736308347.531320]: resetting environment +[INFO] [1736308347.554052]: cleared current plan +[INFO] [1736308347.554323]: moving to home +[INFO] [1736308350.961133]: reached home +[INFO] [1736308350.968969]: reset action history +[INFO] [1736308353.747460]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308360.088139]: resetting environment +[INFO] [1736308360.103140]: cleared current plan +[INFO] [1736308360.107420]: moving to home +[INFO] [1736308361.115266]: reached home +[INFO] [1736308361.121332]: reset action history +[INFO] [1736308363.898562]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_4_2025-01-07-22-52-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:34541/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2247156] +process[print_text-14]: started with pid [2247157] +process[rosbag_record-15]: started with pid [2247158] +started bagging! +[INFO] [1736308392.209172]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736308392.947178]: Stored episode 5. +[INFO] [1736308392.950912]: Saving video... +[WARN] [1736308393.560759]: Plan exhausted +[WARN] [1736308393.604634]: Plan exhausted +[INFO] [1736308393.526381]: Video saved! +Starting episode 5 +[INFO] [1736308393.587818]: resetting environment +[INFO] [1736308393.622155]: cleared current plan +[INFO] [1736308393.624245]: moving to home +[INFO] [1736308397.037478]: reached home +[INFO] [1736308397.039362]: reset action history +[INFO] [1736308399.816535]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308404.816228]: resetting environment +[INFO] [1736308404.836077]: cleared current plan +[INFO] [1736308404.839518]: moving to home +[INFO] [1736308405.851010]: reached home +[INFO] [1736308405.857009]: reset action history +[INFO] [1736308408.630301]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_5_2025-01-07-22-53-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:37575/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-16]: started with pid [2247268] +process[print_text-17]: started with pid [2247269] +process[rosbag_record-18]: started with pid [2247293] +started bagging! +[WARN] [1736308436.960629]: Plan exhausted +[WARN] [1736308437.009187]: Plan exhausted +[INFO] [1736308435.636829]: Storing episode... +[print_text-17] killing on exit +[pub_text-16] killing on exit +[rosbag_record-18] killing on exit +[INFO] [1736308436.393427]: Stored episode 6. +[INFO] [1736308436.405483]: Saving video... +[INFO] [1736308437.112910]: Video saved! +Starting episode 6 +[INFO] [1736308437.179096]: resetting environment +[INFO] [1736308437.190905]: cleared current plan +[INFO] [1736308437.196854]: moving to home +[WARN] [1736308437.040414]: Plan exhausted +[WARN] [1736308437.082261]: Plan exhausted +[WARN] [1736308437.121134]: Plan exhausted +[WARN] [1736308437.161457]: Plan exhausted +[INFO] [1736308440.615038]: reached home +[INFO] [1736308440.621170]: reset action history +[INFO] [1736308443.406153]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308448.661377]: resetting environment +[INFO] [1736308448.685033]: cleared current plan +[INFO] [1736308448.690485]: moving to home +[INFO] [1736308449.704323]: reached home +[INFO] [1736308449.706318]: reset action history +[INFO] [1736308452.450763]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_6_2025-01-07-22-54-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:44535/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-19]: started with pid [2247426] +process[print_text-20]: started with pid [2247427] +process[rosbag_record-21]: started with pid [2247428] +started bagging! +[INFO] [1736308481.128741]: Storing episode... +[pub_text-19] killing on exit[rosbag_record-21] killing on exit +[print_text-20] killing on exit +[INFO] [1736308481.916312]: Stored episode 7. +[INFO] [1736308481.921869]: Saving video... +[INFO] [1736308482.231619]: Video saved! +Starting episode 7 +[INFO] [1736308482.289582]: resetting environment +[INFO] [1736308482.312232]: cleared current plan +[INFO] [1736308482.315091]: moving to home +[INFO] [1736308485.728033]: reached home +[INFO] [1736308485.736361]: reset action history +[INFO] [1736308488.641319]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308494.734448]: resetting environment +[INFO] [1736308494.771841]: cleared current plan +[INFO] [1736308494.774288]: moving to home +[INFO] [1736308495.788469]: reached home +[INFO] [1736308495.812977]: reset action history +[INFO] [1736308498.614125]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_7_2025-01-07-22-54-58.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:40771/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-22]: started with pid [2247539] +process[print_text-23]: started with pid [2247540] +process[rosbag_record-24]: started with pid [2247564] +started bagging! +[WARN] [1736308526.960974]: Plan exhausted +[WARN] [1736308527.002286]: Plan exhausted +[WARN] [1736308527.040376]: Plan exhausted +[INFO] [1736308525.616670]: Storing episode... +[rosbag_record-24] killing on exit[pub_text-22] killing on exit +[print_text-23] killing on exit +[INFO] [1736308526.562172]: Stored episode 8. +[INFO] [1736308526.576902]: Saving video... +[INFO] [1736308526.928576]: Video saved! +Starting episode 8 +[INFO] [1736308527.043408]: resetting environment +[INFO] [1736308527.076503]: cleared current plan +[INFO] [1736308527.080899]: moving to home +[INFO] [1736308530.500184]: reached home +[INFO] [1736308530.513123]: reset action history +[INFO] [1736308533.431538]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308538.594936]: resetting environment +[INFO] [1736308538.631412]: cleared current plan +[INFO] [1736308538.632849]: moving to home +[INFO] [1736308539.650503]: reached home +[INFO] [1736308539.654930]: reset action history +[INFO] [1736308542.543094]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_8_2025-01-07-22-55-42.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:38017/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-25]: started with pid [2247672] +process[print_text-26]: started with pid [2247696] +process[rosbag_record-27]: started with pid [2247697] +started bagging! +For topic gripper_width: timestamp difference is -227429845 for nearest: 1736308546706641934 - target: 1736308546934071779 at idx 53 +For topic gripper_width: timestamp difference is 105618978 for nearest: 1736308547605893875 - target: 1736308547500274897 at idx 61 +[WARN] [1736308570.923649]: Plan exhausted +[WARN] [1736308570.960986]: Plan exhausted +[WARN] [1736308571.001497]: Plan exhausted +[WARN] [1736308571.040819]: Plan exhausted +[WARN] [1736308571.083216]: Plan exhausted +[WARN] [1736308571.120384]: Plan exhausted +[INFO] [1736308569.585636]: Storing episode... +[rosbag_record-27] killing on exit +[print_text-26] killing on exit +[pub_text-25] killing on exit +[INFO] [1736308570.646485]: Stored episode 9. +[INFO] [1736308570.667220]: Saving video... +[INFO] [1736308571.245776]: Video saved! +Starting episode 9 +[INFO] [1736308571.335631]: resetting environment +[INFO] [1736308571.352927]: cleared current plan +[INFO] [1736308571.357387]: moving to home +[WARN] [1736308571.160498]: Plan exhausted +[WARN] [1736308571.204397]: Plan exhausted +[WARN] [1736308571.241588]: Plan exhausted +[WARN] [1736308571.280984]: Plan exhausted +[WARN] [1736308571.320086]: Plan exhausted +[INFO] [1736308574.669808]: reached home +[INFO] [1736308574.682234]: reset action history +[INFO] [1736308577.587786]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308584.028702]: resetting environment +[INFO] [1736308584.073469]: cleared current plan +[INFO] [1736308584.079408]: moving to home +[INFO] [1736308585.089872]: reached home +[INFO] [1736308585.102487]: reset action history +[INFO] [1736308587.974677]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_9_2025-01-07-22-56-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:39189/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-28]: started with pid [2247831] +process[print_text-29]: started with pid [2247832] +process[rosbag_record-30]: started with pid [2247833] +started bagging! +For topic gripper_width: timestamp difference is -182123346 for nearest: 1736308609406835871 - target: 1736308609588959217 at idx 249 +[INFO] [1736308618.856993]: Storing episode... +[pub_text-28] killing on exit +[rosbag_record-30] killing on exit[print_text-29] killing on exit +[INFO] [1736308619.643954]: Stored episode 10. +[INFO] [1736308619.656462]: Saving video... +[INFO] [1736308619.902338]: Video saved! +Starting episode 10 +[INFO] [1736308620.017849]: resetting environment +[INFO] [1736308620.056357]: cleared current plan +[INFO] [1736308620.059986]: moving to home +[INFO] [1736308623.679901]: reached home +[INFO] [1736308623.693769]: reset action history +[INFO] [1736308626.592396]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308635.381681]: resetting environment +[INFO] [1736308635.428359]: cleared current plan +[INFO] [1736308635.441409]: moving to home +[INFO] [1736308636.456811]: reached home +[INFO] [1736308636.458449]: reset action history +[INFO] [1736308639.334437]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_10_2025-01-07-22-57-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:39185/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-31]: started with pid [2247967] +process[print_text-32]: started with pid [2247968] +process[rosbag_record-33]: started with pid [2247969] +started bagging! +[INFO] [1736308669.389232]: Storing episode... +[rosbag_record-33] killing on exit +[pub_text-31] killing on exit +[print_text-32] killing on exit +[INFO] [1736308670.184078]: Stored episode 11. +[INFO] [1736308670.198635]: Saving video... +[INFO] [1736308670.432508]: Video saved! +Starting episode 11 +[INFO] [1736308670.541912]: resetting environment +[INFO] [1736308670.563033]: cleared current plan +[INFO] [1736308670.566639]: moving to home +[INFO] [1736308674.190810]: reached home +[INFO] [1736308674.216868]: reset action history +[INFO] [1736308677.117496]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308681.981911]: resetting environment +[INFO] [1736308682.046482]: cleared current plan +[INFO] [1736308682.051070]: moving to home +[INFO] [1736308683.068047]: reached home +[INFO] [1736308683.088146]: reset action history +[INFO] [1736308686.051574]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_11_2025-01-07-22-58-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:38113/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-34]: started with pid [2248101] +process[print_text-35]: started with pid [2248102] +process[rosbag_record-36]: started with pid [2248103] +started bagging! +[INFO] [1736308718.097479]: Storing episode... +[pub_text-34] killing on exit +[rosbag_record-36] killing on exit +[print_text-35] killing on exit +[INFO] [1736308718.915055]: Stored episode 12. +[INFO] [1736308718.924229]: Saving video... +[INFO] [1736308719.467861]: Video saved! +Starting episode 12 +[INFO] [1736308719.587646]: resetting environment +[INFO] [1736308719.651720]: cleared current plan +[INFO] [1736308719.672147]: moving to home +[WARN] [1736308719.442125]: Plan exhausted +[WARN] [1736308719.481631]: Plan exhausted +[WARN] [1736308719.520624]: Plan exhausted +[WARN] [1736308719.575018]: Plan exhausted +[WARN] [1736308719.601068]: Plan exhausted +[WARN] [1736308719.641244]: Plan exhausted +[INFO] [1736308723.086514]: reached home +[INFO] [1736308723.102974]: reset action history +[INFO] [1736308726.152136]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308736.329606]: resetting environment +[INFO] [1736308736.387248]: cleared current plan +[INFO] [1736308736.406714]: moving to home +[INFO] [1736308737.418729]: reached home +[INFO] [1736308737.433898]: reset action history +[INFO] [1736308740.349713]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_12_2025-01-07-22-59-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:44677/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-37]: started with pid [2248237] +process[print_text-38]: started with pid [2248238] +process[rosbag_record-39]: started with pid [2248239] +started bagging! +For topic gripper_width: timestamp difference is 107001535 for nearest: 1736308755772628061 - target: 1736308755665626526 at idx 195 +For topic gripper_width: timestamp difference is 110374652 for nearest: 1736308760005865775 - target: 1736308759895491123 at idx 249 +[INFO] [1736308772.970737]: Storing episode... +[rosbag_record-39] killing on exit +[pub_text-37] killing on exit +[print_text-38] killing on exit +[WARN] [1736308774.297421]: Plan exhausted +[WARN] [1736308774.328646]: Plan exhausted +[WARN] [1736308774.369625]: Plan exhausted +[WARN] [1736308774.400682]: Plan exhausted +[WARN] [1736308774.440729]: Plan exhausted +[WARN] [1736308774.481267]: Plan exhausted +[WARN] [1736308774.523306]: Plan exhausted +[WARN] [1736308774.560847]: Plan exhausted +[WARN] [1736308774.600095]: Plan exhausted +[WARN] [1736308774.641826]: Plan exhausted +[WARN] [1736308774.684678]: Plan exhausted +[WARN] [1736308774.721287]: Plan exhausted +[INFO] [1736308773.817849]: Stored episode 13. +[INFO] [1736308773.847095]: Saving video... +[INFO] [1736308774.535989]: Video saved! +Starting episode 13 +[INFO] [1736308774.702979]: resetting environment +[INFO] [1736308774.734344]: cleared current plan +[INFO] [1736308774.739225]: moving to home +[INFO] [1736308778.254585]: reached home +[INFO] [1736308778.270283]: reset action history +[INFO] [1736308781.246950]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308784.490631]: resetting environment +[INFO] [1736308784.529728]: cleared current plan +[INFO] [1736308784.532573]: moving to home +[INFO] [1736308785.554098]: reached home +[INFO] [1736308785.585304]: reset action history +[INFO] [1736308788.576977]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_13_2025-01-07-22-59-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:36837/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-40]: started with pid [2248350] +process[print_text-41]: started with pid [2248374] +process[rosbag_record-42]: started with pid [2248375] +started bagging! +[INFO] [1736308815.624735]: Storing episode... +[pub_text-40] killing on exit +[print_text-41] killing on exit +[rosbag_record-42] killing on exit +[WARN] [1736308816.961399]: Plan exhausted +[WARN] [1736308817.004985]: Plan exhausted +[WARN] [1736308817.041826]: Plan exhausted +[WARN] [1736308817.080626]: Plan exhausted +[WARN] [1736308817.137851]: Plan exhausted +[WARN] [1736308817.170759]: Plan exhausted +[WARN] [1736308817.200276]: Plan exhausted +[INFO] [1736308816.448116]: Stored episode 14. +[INFO] [1736308816.456821]: Saving video... +[INFO] [1736308817.058468]: Video saved! +Starting episode 14 +[INFO] [1736308817.176826]: resetting environment +[INFO] [1736308817.218090]: cleared current plan +[INFO] [1736308817.219251]: moving to home +[INFO] [1736308820.637812]: reached home +[INFO] [1736308820.641756]: reset action history +[INFO] [1736308823.678071]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308830.162459]: resetting environment +[INFO] [1736308830.197794]: cleared current plan +[INFO] [1736308830.201463]: moving to home +[INFO] [1736308831.213885]: reached home +[INFO] [1736308831.228844]: reset action history +[INFO] [1736308834.303280]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_14_2025-01-07-23-00-34.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:36601/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-43]: started with pid [2248512] +process[print_text-44]: started with pid [2248513] +process[rosbag_record-45]: started with pid [2248514] +started bagging! +[WARN] [1736308869.363117]: Plan exhausted +[WARN] [1736308869.404453]: Plan exhausted +[INFO] [1736308868.035090]: Storing episode... +[print_text-44] killing on exit +[pub_text-43] killing on exit[rosbag_record-45] killing on exit +[INFO] [1736308868.864231]: Stored episode 15. +[INFO] [1736308868.866849]: Saving video... +[INFO] [1736308869.582262]: Video saved! +Starting episode 15 +[INFO] [1736308869.692563]: resetting environment +[INFO] [1736308869.740857]: cleared current plan +[INFO] [1736308869.760037]: moving to home +[WARN] [1736308869.440470]: Plan exhausted +[WARN] [1736308869.482149]: Plan exhausted +[WARN] [1736308869.520121]: Plan exhausted +[WARN] [1736308869.561620]: Plan exhausted +[WARN] [1736308869.600101]: Plan exhausted +[WARN] [1736308869.651290]: Plan exhausted +[WARN] [1736308869.699917]: Plan exhausted +[WARN] [1736308869.720092]: Plan exhausted +[INFO] [1736308873.097802]: reached home +[INFO] [1736308873.121998]: reset action history +[INFO] [1736308876.112684]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308889.730684]: resetting environment +[INFO] [1736308889.782757]: cleared current plan +[INFO] [1736308889.801387]: moving to home +[INFO] [1736308890.815609]: reached home +[INFO] [1736308890.819956]: reset action history +[INFO] [1736308893.853673]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_15_2025-01-07-23-01-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:37133/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-46]: started with pid [2248648] +process[print_text-47]: started with pid [2248649] +process[rosbag_record-48]: started with pid [2248650] +started bagging! +For topic gripper_width: timestamp difference is -105194965 for nearest: 1736308905605861744 - target: 1736308905711056709 at idx 170 +[INFO] [1736308926.778728]: Storing episode... +[pub_text-46] killing on exit +[rosbag_record-48] killing on exit +[print_text-47] killing on exit +[INFO] [1736308927.577839]: Stored episode 16. +[INFO] [1736308927.592519]: Saving video... +[WARN] [1736308928.129788]: Plan exhausted +[WARN] [1736308928.163839]: Plan exhausted +[WARN] [1736308928.206002]: Plan exhausted +[WARN] [1736308928.240125]: Plan exhausted +[WARN] [1736308928.281124]: Plan exhausted +[INFO] [1736308928.097557]: Video saved! +Starting episode 16 +[INFO] [1736308928.252420]: resetting environment +[INFO] [1736308928.291203]: cleared current plan +[INFO] [1736308928.292434]: moving to home +[INFO] [1736308931.908306]: reached home +[INFO] [1736308931.917782]: reset action history +[INFO] [1736308935.021137]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308944.097084]: resetting environment +[INFO] [1736308944.212710]: cleared current plan +[INFO] [1736308944.227375]: moving to home +[INFO] [1736308945.260417]: reached home +[INFO] [1736308945.275574]: reset action history +[INFO] [1736308948.392509]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_16_2025-01-07-23-02-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:44921/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-49]: started with pid [2248784] +process[print_text-50]: started with pid [2248785] +process[rosbag_record-51]: started with pid [2248786] +started bagging! +For topic gripper_width: timestamp difference is -253584353 for nearest: 1736308971505892309 - target: 1736308971759476662 at idx 249 +[INFO] [1736308983.240807]: Storing episode... +[rosbag_record-51] killing on exit +[pub_text-49] killing on exit +[print_text-50] killing on exit +[WARN] [1736308984.560521]: Plan exhausted +[WARN] [1736308984.605591]: Plan exhausted +[WARN] [1736308984.648594]: Plan exhausted +[WARN] [1736308984.680778]: Plan exhausted +[INFO] [1736308984.094237]: Stored episode 17. +[INFO] [1736308984.103043]: Saving video... +[INFO] [1736308984.491891]: Video saved! +Starting episode 17 +[INFO] [1736308984.636433]: resetting environment +[INFO] [1736308984.705058]: cleared current plan +[INFO] [1736308984.719465]: moving to home +[INFO] [1736308988.351489]: reached home +[INFO] [1736308988.364748]: reset action history +[INFO] [1736308991.493768]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736308998.068830]: resetting environment +[INFO] [1736308998.123169]: cleared current plan +[INFO] [1736308998.142502]: moving to home +[INFO] [1736308999.157624]: reached home +[INFO] [1736308999.166329]: reset action history +[INFO] [1736309002.245214]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_17_2025-01-07-23-03-22.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:40303/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-52]: started with pid [2248920] +process[print_text-53]: started with pid [2248921] +process[rosbag_record-54]: started with pid [2248922] +started bagging! +[INFO] [1736309037.229960]: Storing episode... +[pub_text-52] killing on exit +[rosbag_record-54] killing on exit +[print_text-53] killing on exit +[INFO] [1736309037.977221]: Stored episode 18. +[INFO] [1736309038.006693]: Saving video... +[WARN] [1736309038.570376]: Plan exhausted +[WARN] [1736309038.618741]: Plan exhausted +[INFO] [1736309038.432548]: Video saved! +Starting episode 18 +[INFO] [1736309038.590930]: resetting environment +[INFO] [1736309038.640284]: cleared current plan +[INFO] [1736309038.641402]: moving to home +[INFO] [1736309042.253412]: reached home +[INFO] [1736309042.257095]: reset action history +[INFO] [1736309045.531005]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309051.977226]: resetting environment +[INFO] [1736309052.012718]: cleared current plan +[INFO] [1736309052.012925]: moving to home +[INFO] [1736309053.015294]: reached home +[INFO] [1736309053.020332]: reset action history +[INFO] [1736309056.143846]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_18_2025-01-07-23-04-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:35123/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-55]: started with pid [2249056] +process[print_text-56]: started with pid [2249057] +process[rosbag_record-57]: started with pid [2249058] +started bagging! +For topic gripper_width: timestamp difference is 129667258 for nearest: 1736309068872551417 - target: 1736309068742884159 at idx 180 +For topic gripper_width: timestamp difference is 131299739 for nearest: 1736309072072524076 - target: 1736309071941224337 at idx 224 +For topic gripper_width: timestamp difference is 101826168 for nearest: 1736309079672496296 - target: 1736309079570670128 at idx 249 +For topic gripper_width: timestamp difference is -226898201 for nearest: 1736309085005860083 - target: 1736309085232758284 at idx 249 +For topic gripper_width: timestamp difference is 106861877 for nearest: 1736309085905823516 - target: 1736309085798961639 at idx 249 +[INFO] [1736309092.059557]: Storing episode... +[print_text-56] killing on exit +[rosbag_record-57] killing on exit +[pub_text-55] killing on exit +[WARN] [1736309093.366179]: Plan exhausted +[WARN] [1736309093.402355]: Plan exhausted +[WARN] [1736309093.442347]: Plan exhausted +[WARN] [1736309093.486063]: Plan exhausted +[WARN] [1736309093.522272]: Plan exhausted +[WARN] [1736309093.572426]: Plan exhausted +[WARN] [1736309093.607945]: Plan exhausted +[WARN] [1736309093.642984]: Plan exhausted +[INFO] [1736309092.906763]: Stored episode 19. +[INFO] [1736309092.936285]: Saving video... +[INFO] [1736309093.469589]: Video saved! +Starting episode 19 +[INFO] [1736309093.673985]: resetting environment +[INFO] [1736309093.722153]: cleared current plan +[INFO] [1736309093.722388]: moving to home +[WARN] [1736309093.689347]: Plan exhausted +[WARN] [1736309093.720437]: Plan exhausted +[INFO] [1736309097.333074]: reached home +[INFO] [1736309097.349892]: reset action history +[INFO] [1736309100.587520]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736309104.092758]: resetting environment +[INFO] [1736309104.173210]: cleared current plan +[INFO] [1736309104.205497]: moving to home +[INFO] [1736309105.212814]: reached home +[INFO] [1736309105.238891]: reset action history +[INFO] [1736309108.324866]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/episode_rosbags/episode_19_2025-01-07-23-05-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2246319.log +started roslaunch server http://158.130.50.37:41977/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-58]: started with pid [2249166] +process[print_text-59]: started with pid [2249190] +process[rosbag_record-60]: started with pid [2249191] +started bagging! +For topic gripper_width: timestamp difference is 101432099 for nearest: 1736309120372528329 - target: 1736309120271096230 at idx 180 +For topic gripper_width: timestamp difference is 101971413 for nearest: 1736309121972558047 - target: 1736309121870586634 at idx 202 +For topic gripper_width: timestamp difference is 103469223 for nearest: 1736309123572524160 - target: 1736309123469054937 at idx 223 +For topic gripper_width: timestamp difference is 113307526 for nearest: 1736309134805860569 - target: 1736309134692553043 at idx 249 +[INFO] [1736309135.368889]: Storing episode... +[WARN] [1736309136.724657]: Plan exhausted +[WARN] [1736309136.760630]: Plan exhausted +[WARN] [1736309136.801356]: Plan exhausted +[WARN] [1736309136.848559]: Plan exhausted +[WARN] [1736309136.883331]: Plan exhausted +[WARN] [1736309136.927868]: Plan exhausted +[WARN] [1736309136.982156]: Plan exhausted +[WARN] [1736309137.008655]: Plan exhausted +[WARN] [1736309137.040113]: Plan exhausted +[WARN] [1736309137.087768]: Plan exhausted +[WARN] [1736309137.124140]: Plan exhausted +[WARN] [1736309137.172403]: Plan exhausted +[WARN] [1736309137.217685]: Plan exhausted +[WARN] [1736309137.256990]: Plan exhausted +[WARN] [1736309137.302410]: Plan exhausted +[WARN] [1736309137.328058]: Plan exhausted +[WARN] [1736309137.361567]: Plan exhausted +[WARN] [1736309137.409573]: Plan exhausted +[WARN] [1736309137.441500]: Plan exhausted +[WARN] [1736309137.490631]: Plan exhausted +[WARN] [1736309137.521251]: Plan exhausted +[WARN] [1736309137.560861]: Plan exhausted +[WARN] [1736309137.610563]: Plan exhausted +[WARN] [1736309137.648181]: Plan exhausted +[WARN] [1736309137.693516]: Plan exhausted +[print_text-59] killing on exit +[pub_text-58] killing on exit[rosbag_record-60] killing on exit +[INFO] [1736309137.251199]: Stored episode 20. +[INFO] [1736309137.287743]: Saving video... +[INFO] [1736309138.016223]: Video saved! +[INFO] [1736309138.151665]: resetting environment +[INFO] [1736309138.183651]: cleared current plan +[INFO] [1736309138.186501]: moving to home +[WARN] [1736309137.729559]: Plan exhausted +[WARN] [1736309137.760373]: Plan exhausted +[WARN] [1736309137.815898]: Plan exhausted +[WARN] [1736309137.840913]: Plan exhausted +[WARN] [1736309137.880112]: Plan exhausted +[WARN] [1736309137.921413]: Plan exhausted +[WARN] [1736309137.960138]: Plan exhausted +[WARN] [1736309138.000873]: Plan exhausted +[WARN] [1736309138.043110]: Plan exhausted +[WARN] [1736309138.080098]: Plan exhausted +[WARN] [1736309138.121360]: Plan exhausted +[WARN] [1736309138.165558]: Plan exhausted +[INFO] [1736309141.794493]: reached home +[INFO] [1736309141.812051]: reset action history +[INFO] [1736309144.937301]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4130c5211c09573797d8ddd25a8f37b9c34c3e54 --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T03:49:27.159508", + "startedAt": "2025-01-08T03:49:26.768531", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3747.1576250000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4487.143, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 710.4931983947754 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..852a3e734ad31f0cc41eca9b000a05d134a5459e --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "8a7acf7610c7a7f0d62e8d433ce287a8947ce47c8f1195c93b999c25ecce79b3", "size": 1138909, "path": "media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4"}, "global_step": 2660, "_timestamp": 1736309151.2540703, "_runtime": 984.4762382507324, "_step": 41, "eval/1_eval": {"_type": "video-file", "sha256": "6105fa800baa66794d18fe8647635294b8fd467674d9a85bfd6647039cf1293d", "size": 1193600, "path": "media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4"}, "eval/num_success": 6.0, "episode": 145.0, "eval/success_rate": 0.30000001192092896, "eval/success_rate_lb_0.01": 0.10039830207824707, "eval/success_rate_ub_0.01": 0.5585116744041443, "eval/success_rate_lb_0.025": 0.12152326852083206, "eval/success_rate_ub_0.025": 0.5025138854980469, "eval/success_rate_lb_0.05": 0.1669653207063675, "eval/success_rate_ub_0.05": 0.48316776752471924, "eval/success_rate_lb_0.1": 0.1789700835943222, "eval/success_rate_ub_0.1": 0.4645809233188629, "eval/2_eval": {"_type": "video-file", "sha256": "c92292561966baf2ee3899d136adc87f373197924f837a6a3dddb13dd7dfc730", "size": 1143421, "path": "media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4978aa6435b32ecf67343f7f22cb866f6a82d85d5d1892976fb019fffb095a08", "size": 1200135, "path": "media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ecf6c04acd951daf42e0863e775e650eb62d0fba1c2c91368af151a2718d6a42", "size": 1247071, "path": "media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4"}, "eval/5_eval": {"_type": "video-file", "sha256": "c452dd651b0df318063859d846c3dd0082130d16fe37e52f60da91cffdecb788", "size": 1198593, "path": "media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4"}, "eval/6_eval": {"_type": "video-file", "sha256": "a40bb3e24616a11f81cf8f8cb531134b91ca09b4d31f5c66d26081f8a597697a", "size": 1243138, "path": "media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4"}, "eval/7_eval": {"_type": "video-file", "sha256": "1822051540a8ae17e31542973fd9483b3e04543b61fa35aa2e112db4e7930691", "size": 1216136, "path": "media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4"}, "eval/8_eval": {"_type": "video-file", "sha256": "649fa438a50ad92ff033f4db32a883231540f22a68f0552f52a39c5668b78980", "size": 1201391, "path": "media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4"}, "eval/9_eval": {"_type": "video-file", "sha256": "ae51d826a95728d28cae6b1b8c551126c642ebe2d2618f0dd4e5d49b45ec61d6", "size": 1280834, "path": "media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4"}, "eval/10_eval": {"_type": "video-file", "sha256": "1c5570816ad9da40a43e521c2692fbf3c896a7868112e102639b1f803807b568", "size": 1278356, "path": "media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4"}, "eval/11_eval": {"_type": "video-file", "sha256": "a606d9c48c91655df31c54ec48b57edcb5ae66c9e0e72e08d0040360683cbca7", "size": 1399219, "path": "media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4"}, "eval/12_eval": {"_type": "video-file", "sha256": "3a187eeb09299696aa2a1651b0c5885f8d4888b203fafb6872a4451f7c71ffef", "size": 1427340, "path": "media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4"}, "eval/13_eval": {"_type": "video-file", "sha256": "9efca5d983ee670cf985545f839f820290fa9648c2efe9805271496b3e2afe5c", "size": 1205126, "path": "media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4"}, "eval/14_eval": {"_type": "video-file", "sha256": "ffa92cd8767b9ad59d685b39c030e4373019479e38dcd00037ea91346b07353d", "size": 1373307, "path": "media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4"}, "eval/15_eval": {"_type": "video-file", "sha256": "af536578f4476ce23195e3698f30ff9f31c2321e1d8a9efb7a5d698910722d35", "size": 1358907, "path": "media/videos/eval/15_eval_15_af536578f4476ce23195.mp4"}, "eval/16_eval": {"_type": "video-file", "sha256": "db31934236335d9e75e4f0c34cec84e72d542118c7c4ee6ce4f5de77075484b3", "size": 1444676, "path": "media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4"}, "eval/17_eval": {"_type": "video-file", "sha256": "a0293b276af8b7ee82ad1af37fc416a012c67d2159e42b28d61210c141cd56c7", "size": 1445556, "path": "media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4"}, "eval/18_eval": {"_type": "video-file", "sha256": "205c297dae90682b6f426123d9088c72777d0add28b23f8d2ff0181423bf5020", "size": 1459635, "path": "media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4"}, "eval/19_eval": {"_type": "video-file", "sha256": "27b3cf56f899624bcd38b38ef615d544991af40aab002b25fe8e00be0095a8ca", "size": 1191669, "path": "media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "97faedb494e69b73017ebacc893b65c2a0c945512cc9e63c1939b212ee9763ad", "size": 399, "artifact_path": "wandb-client-artifact://c8g29bx1nefkpsr3gl88rgsfs9coyzhfdr2ofrq236ylrs3v39bg68ui5fm3srgkvcfwljkc0ezcczm1k8ek930x9uyylceakrqthx8t9nzhh8lgsudzu4sn9fn74woo/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://uerpy5757l30cfbd1c71wwhdvu91vdww3c15obs0espb8b6yby86qh2okl121k0gfcy6esga7z9h8xk6b0nt4it7e080ho95nzh90rnjf4x4v5g2hwdly9eki2e9v9s2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 47, "total_num_episodes": 145, "total_success_rate": 0.32413793103448274, "total_success_rate_lb_0.01": 0.24027697747179777, "total_success_rate_ub_0.01": 0.41904157401750397, "total_success_rate_lb_0.025": 0.25318085779715693, "total_success_rate_ub_0.025": 0.40072802456218604, "total_success_rate_lb_0.05": 0.26618888606336943, "total_success_rate_ub_0.05": 0.3878559880182919, "total_success_rate_lb_0.1": 0.2730443329072916, "total_success_rate_ub_0.1": 0.3766404036679225, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "0e761adafea3a3b0973611d3358b30eef511c6275d4df46b20926f5703987ef8", "size": 527, "artifact_path": "wandb-client-artifact://at0yjnhsn24m3vaargj12bkdaggo63qeh8clg09j5kblaghxb4ny77iiiu3jfyt9v1w39mmud2qpwo0xzgan8hiozbjficw4dox5sqy90ujchpj8vkxos0hgtmy89j5z/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tbcix8a009depxemyoomcz6i5dlmrzbjsfwjgluam7j09n0qm7bixstdu0zc3cok9eceqhx4qh7nzi9qurn1ld0n33cqx14q7lqpu4gf5nxqpffbzxjoo0jzvizhqu7h:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 1014}, "eval/total_success_rate": 0.32413792610168457, "eval/total_success_rate_lb_0.01": 0.2402769774198532, "eval/total_success_rate_ub_0.01": 0.41904157400131226, "eval/total_success_rate_lb_0.025": 0.2531808614730835, "eval/total_success_rate_ub_0.025": 0.4007280170917511, "eval/total_success_rate_lb_0.05": 0.2661888897418976, "eval/total_success_rate_ub_0.05": 0.38785597681999207, "eval/total_success_rate_lb_0.1": 0.2730443477630615, "eval/total_success_rate_ub_0.1": 0.3766404092311859} \ No newline at end of file diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug-internal.log b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..412522e0e105a2faf505a71babdfcb7902acf7ba --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug-internal.log @@ -0,0 +1,2171 @@ +2025-01-07 22:49:26,777 INFO StreamThr :2246486 [internal.py:wandb_internal():85] W&B internal server running at pid: 2246486, started at: 2025-01-07 22:49:26.776644 +2025-01-07 22:49:26,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status +2025-01-07 22:49:26,788 INFO WriterThread:2246486 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb +2025-01-07 22:49:26,789 DEBUG SenderThread:2246486 [sender.py:send():379] send: header +2025-01-07 22:49:26,798 DEBUG SenderThread:2246486 [sender.py:send():379] send: run +2025-01-07 22:49:27,040 INFO SenderThread:2246486 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files +2025-01-07 22:49:27,040 INFO SenderThread:2246486 [sender.py:_start_run_threads():1188] run started: bmk18y9z with start time 1736308166.777832 +2025-01-07 22:49:27,054 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 22:49:27,054 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: check_version +2025-01-07 22:49:27,141 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 22:49:27,146 DEBUG HandlerThread:2246486 [system_info.py:__init__():26] System info init +2025-01-07 22:49:27,146 DEBUG HandlerThread:2246486 [system_info.py:__init__():41] System info init done +2025-01-07 22:49:27,146 INFO HandlerThread:2246486 [system_monitor.py:start():194] Starting system monitor +2025-01-07 22:49:27,146 INFO SystemMonitor:2246486 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 22:49:27,147 INFO HandlerThread:2246486 [system_monitor.py:probe():214] Collecting system info +2025-01-07 22:49:27,147 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 22:49:27,147 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started disk monitoring +2025-01-07 22:49:27,148 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 22:49:27,148 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started memory monitoring +2025-01-07 22:49:27,149 INFO SystemMonitor:2246486 [interfaces.py:start():188] Started network monitoring +2025-01-07 22:49:27,159 DEBUG HandlerThread:2246486 [system_info.py:probe():152] Probing system +2025-01-07 22:49:27,160 DEBUG HandlerThread:2246486 [system_info.py:_probe_git():137] Probing git +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:_probe_git():145] Probing git done +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:probe():200] Probing system done +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T03:49:27.159508', 'startedAt': '2025-01-08T03:49:26.768531', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.1576250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4487.143, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 710.4931983947754}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 22:49:27,165 INFO HandlerThread:2246486 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 22:49:27,165 INFO HandlerThread:2246486 [system_monitor.py:probe():227] Publishing system info +2025-01-07 22:49:27,165 DEBUG HandlerThread:2246486 [system_info.py:_save_code():44] Saving code +2025-01-07 22:49:27,171 DEBUG HandlerThread:2246486 [system_info.py:_save_code():67] Saving code done +2025-01-07 22:49:27,171 DEBUG HandlerThread:2246486 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 22:49:27,241 DEBUG HandlerThread:2246486 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 22:49:27,241 INFO HandlerThread:2246486 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 22:49:27,245 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 22:49:27,245 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 22:49:27,482 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/i43982w5-wandb-metadata.json +2025-01-07 22:49:27,501 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4rgavcbr-diff.patch +2025-01-07 22:49:27,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 22:49:27,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:27,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: python_packages +2025-01-07 22:49:27,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:27,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:27,509 INFO HandlerThread:2246486 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/tb" + save: true +} +control { + relay_id: "870c06ac45b14de49a3ec250a2529b53" +} +_info { + stream_id: "bmk18y9z" +} + +2025-01-07 22:49:27,509 DEBUG HandlerThread:2246486 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 22:49:27,541 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xahuuotc-code/FISH/eval_robot.py +2025-01-07 22:49:27,603 DEBUG SenderThread:2246486 [sender.py:send():379] send: telemetry +2025-01-07 22:49:27,604 DEBUG SenderThread:2246486 [sender.py:send():379] send: tbrecord +2025-01-07 22:49:27,604 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:49:27,604 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 with policy live +2025-01-07 22:49:27,793 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4qb76c3i-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:49:28,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code +2025-01-07 22:49:28,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH +2025-01-07 22:49:28,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:29,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:30,041 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:30,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:31,315 DEBUG SenderThread:2246486 [sender.py:send():379] send: config +2025-01-07 22:49:31,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:32,326 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:32,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:33,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:34,042 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:34,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:35,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:36,043 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:36,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:37,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:37,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:38,044 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:38,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:39,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:40,044 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:40,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:41,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:42,045 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:42,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:42,506 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:42,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:42,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:43,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:44,046 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:44,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:45,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:46,046 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:49:46,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:47,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:48,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:48,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:49,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:50,509 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:51,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:52,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:53,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:54,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:49:54,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:55,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:56,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:57,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:49:57,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:49:57,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:58,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:59,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:49:59,663 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:00,050 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 22:50:00,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:01,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:02,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:03,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:04,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:05,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:05,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:06,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:07,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:08,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:09,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:10,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:11,192 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:11,517 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:12,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:50:12,158 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 with policy now +2025-01-07 22:50:12,162 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:50:12,163 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:50:12,163 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:50:12,164 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:50:12,408 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/nrhyzaq0-media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:12,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:12,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:12,517 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:13,053 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:50:13,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media +2025-01-07 22:50:13,343 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/2nkbwpip-media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 22:50:13,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:14,054 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:14,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:15,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:16,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:16,817 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:17,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:18,055 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:18,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:19,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:20,056 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:20,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:21,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:22,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:22,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:23,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:24,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:25,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:26,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:27,058 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:50:27,149 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 22:50:27,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:50:27,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:27,507 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:27,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:27,608 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:28,059 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:28,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:29,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:30,059 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:30,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:31,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:32,060 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:32,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:33,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:33,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:34,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:35,521 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:36,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:37,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:38,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:39,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:39,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:40,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:41,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:42,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:42,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:42,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:43,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:44,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:44,642 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:45,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:46,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:47,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:48,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:49,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:50,514 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:50,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:51,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:52,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:53,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:54,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:55,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:56,068 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:56,084 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:50:56,525 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:57,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:50:57,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:50:57,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:50:57,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:58,068 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:50:58,293 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:50:58,293 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 with policy now +2025-01-07 22:50:58,293 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:50:58,295 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:50:58,296 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:50:58,296 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:50:58,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:50:58,586 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/3zppzal7-media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:50:59,352 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/fhhjjeou-media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 22:50:59,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:00,069 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:00,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:01,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:01,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:02,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:03,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:04,070 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:04,541 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:05,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:06,071 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:06,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:06,542 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:07,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:08,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:09,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:10,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:11,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:11,329 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4of8106r-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:11,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:11,842 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:11,909 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:11,909 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:11,909 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:12,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:12,073 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:12,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:12,509 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:12,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:13,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:14,074 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:14,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:15,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:16,075 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:16,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:16,940 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:17,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:18,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:19,529 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:20,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:21,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:21,941 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:22,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:23,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:24,077 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:24,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:25,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:26,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:26,946 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:27,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:51:27,508 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:27,508 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:27,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:28,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:29,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:30,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:31,532 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:32,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:32,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:33,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:34,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:35,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:36,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:37,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:37,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:38,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:39,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:40,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:41,533 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:42,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:42,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:42,511 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:42,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:42,665 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:42,809 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:51:42,809 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:51:42,809 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 with policy now +2025-01-07 22:51:42,810 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:42,810 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:42,810 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:43,068 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/09he7tcy-media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:51:43,344 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/54vbj6wu-media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 22:51:43,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:44,084 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:44,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:45,540 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:46,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:47,544 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:48,086 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:48,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:48,545 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:49,544 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:50,086 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:50,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:51,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:52,556 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:53,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:54,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:51:54,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:55,088 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:55,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:55,741 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:51:55,741 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:51:55,741 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:51:56,088 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:51:56,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:51:56,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:56,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:57,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:51:57,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:51:57,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:51:57,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:58,089 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:51:58,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:51:59,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:00,090 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:00,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:00,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:01,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:02,090 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:02,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:03,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:04,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:05,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:05,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:06,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:07,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:08,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:09,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:10,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:11,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:11,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:12,511 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:12,511 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:12,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:13,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:14,552 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:15,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:16,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:16,624 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:17,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:18,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:19,553 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:20,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:21,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:22,519 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:22,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:23,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:24,098 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:24,554 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:25,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:26,557 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:27,151 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:52:27,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:27,516 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:27,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:52:27,558 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:52:27,600 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 with policy now +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:52:27,600 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:52:27,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:27,601 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:52:27,862 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/dy3n98w3-media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:52:28,099 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,100 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:52:28,376 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4tof1kf1-media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 22:52:28,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:29,563 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:30,100 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:30,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:31,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:32,101 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:32,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:33,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:33,565 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:34,567 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:35,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:36,102 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:36,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:37,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:38,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:39,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:39,569 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:40,103 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:52:40,364 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/ygnz6olk-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:52:40,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:40,799 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:52:40,799 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:52:40,800 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:52:41,104 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:52:41,570 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:42,104 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:42,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:42,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:42,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:43,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:44,581 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:44,945 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:45,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:46,106 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:46,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:47,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:48,106 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:52:48,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:49,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:50,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:50,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:51,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:52,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:53,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:54,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:55,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:52:55,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:56,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:57,152 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:52:57,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:52:57,513 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:52:57,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:58,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:52:59,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:00,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:00,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:01,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:02,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:03,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:04,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:05,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:06,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:06,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:07,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:08,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:09,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:10,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:11,575 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:12,209 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:12,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:12,518 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:12,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:13,580 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:53:13,580 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 with policy now +2025-01-07 22:53:13,580 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:13,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:53:13,583 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:13,583 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:13,584 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:13,831 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/pcomtarx-media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:14,115 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:53:14,405 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/swjopcct-media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 22:53:14,576 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:15,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:16,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:16,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:17,528 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:17,585 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:18,116 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:18,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:19,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:20,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:21,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:22,118 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:22,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:22,596 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:23,596 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:24,601 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:24,850 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:24,851 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:24,852 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:25,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:53:25,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:25,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:26,119 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:26,593 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:27,152 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:53:27,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:27,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:27,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:27,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:28,120 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:28,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:29,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:30,121 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:30,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:31,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:32,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:33,526 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:33,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:34,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:35,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:36,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:37,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:38,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:38,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:39,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:40,595 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:41,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:42,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:42,515 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:42,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:43,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:43,608 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:44,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:45,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:46,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:47,612 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:48,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:49,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:49,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:50,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:51,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:52,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:53,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:54,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:53:54,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:55,614 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:56,615 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:57,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:53:57,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:53:57,154 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 with policy now +2025-01-07 22:53:57,162 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:53:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:53:57,162 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:53:57,163 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:53:57,417 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/qm8tagd0-media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:57,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:53:57,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:53:57,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:58,130 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:58,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:53:58,431 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/u3i0jw7h-media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 22:53:58,618 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:59,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:53:59,624 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:00,131 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:00,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:01,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:02,132 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:02,626 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:03,619 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:04,133 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:04,632 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:05,449 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:05,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:06,133 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:06,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:07,622 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:08,649 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:08,907 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:08,907 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:08,908 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:09,134 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:09,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:09,421 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/6aw8p6vy-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:09,626 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:10,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:10,632 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:10,745 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:11,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:12,135 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:12,518 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:12,518 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:12,627 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:13,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:14,136 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:14,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:15,627 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:16,137 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:16,414 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:16,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:17,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:18,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:19,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:20,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:21,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:21,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:22,628 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:23,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:24,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:25,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:26,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:26,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:27,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:54:27,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:27,516 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:27,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:28,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:29,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:30,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:31,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:31,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:32,629 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:33,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:34,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:35,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:36,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:37,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:37,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:38,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:39,630 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:40,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:41,640 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:42,147 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:42,280 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:54:42,280 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 with policy now +2025-01-07 22:54:42,281 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:54:42,282 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:42,283 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:42,284 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:42,520 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:42,520 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:42,546 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/e4pyyz5y-media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:42,591 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:42,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:43,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:54:43,454 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/q99puyrq-media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 22:54:43,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:44,148 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:44,640 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:45,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:46,651 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:47,655 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:48,150 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:48,540 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:48,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:49,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:50,150 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:50,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:51,653 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:52,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:53,543 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:53,657 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:54,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:54,966 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:54:54,966 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:54:54,967 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:54:55,152 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:54:55,152 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:54:55,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:56,153 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:56,653 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:57,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:54:57,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:54:57,535 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:54:57,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:58,153 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:54:58,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:54:58,669 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:54:59,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:00,154 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:00,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:01,646 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:02,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:03,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:04,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:04,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:05,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:06,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:07,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:08,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:09,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:09,647 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:10,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:11,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:12,522 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:12,522 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:12,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:13,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:14,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:14,658 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:15,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:16,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:17,648 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:18,649 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:19,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:20,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:20,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:21,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:22,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:23,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:24,650 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:25,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:25,657 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:26,666 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:27,021 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:55:27,021 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 with policy now +2025-01-07 22:55:27,031 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:55:27,031 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:55:27,031 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:55:27,032 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:55:27,154 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:55:27,275 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/u89v0n8x-media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:27,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:27,548 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7d1rxucs-media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 22:55:27,686 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:28,164 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:28,664 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:29,669 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:30,545 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:30,665 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:31,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:32,166 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:32,668 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:33,662 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:34,166 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:34,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:35,674 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:36,477 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:36,674 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:37,670 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:38,666 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:39,018 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:55:39,018 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:55:39,018 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:55:39,168 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:55:39,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:55:39,421 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7vbnlg3i-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:55:39,693 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:40,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:40,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:41,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:41,693 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:42,169 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:42,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:42,532 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:42,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:43,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:44,170 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:44,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:45,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:46,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:46,867 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:47,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:48,172 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:48,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:49,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:50,173 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:55:50,682 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:51,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:52,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:52,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:53,683 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:54,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:55,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:56,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:57,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:55:57,523 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:55:57,523 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:55:57,634 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:55:57,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:58,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:55:59,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:00,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:01,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:02,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:03,537 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:03,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:04,684 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:05,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:06,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:07,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:08,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:08,685 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:09,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:10,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:11,317 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:56:11,317 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 with policy now +2025-01-07 22:56:11,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:56:11,320 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:56:11,320 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:56:11,320 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:56:11,565 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/3c60a9nl-media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:11,694 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:56:12,181 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:12,182 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:56:12,469 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xpqoq56m-media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 22:56:12,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:12,539 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:12,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:13,695 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:13,695 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:14,182 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:14,717 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:15,702 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:16,183 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:16,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:17,699 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:18,726 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:19,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:19,715 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:20,184 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:20,730 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:21,725 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:22,729 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:23,734 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:24,072 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:56:24,072 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:56:24,073 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:56:24,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:56:24,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:56:24,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:25,080 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:25,745 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:26,186 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:26,750 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:27,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:56:27,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:27,534 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:27,734 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:28,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:29,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:30,188 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:30,735 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:31,017 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:31,755 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:32,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:33,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:34,190 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:34,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:35,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:36,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:36,739 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:37,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:38,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:39,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:40,741 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:41,549 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:41,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:42,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:42,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:42,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:43,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:44,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:45,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:46,611 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:46,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:47,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:48,742 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:49,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:50,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:51,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:51,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:52,197 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:56:52,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:53,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:54,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:55,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:56,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:57,155 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:56:57,156 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:56:57,527 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:56:57,527 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:56:57,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:58,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:56:59,754 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:00,008 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:57:00,008 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 with policy now +2025-01-07 22:57:00,012 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:57:00,013 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:00,013 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:00,013 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,200 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:57:00,286 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/soubxlgx-media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,554 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gmbe9tvy-media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 22:57:00,760 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:01,764 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:02,201 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:02,561 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:02,758 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:03,770 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:04,766 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:05,767 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:06,202 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:06,762 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:07,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:07,775 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:08,203 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:08,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:09,781 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:10,778 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:11,772 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:12,546 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:12,546 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:12,694 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:12,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:13,793 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:14,785 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:15,798 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:16,136 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:16,137 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:16,137 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:16,206 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:57:16,457 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mlbpxsy1-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:57:16,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:17,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:18,208 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:18,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:18,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:19,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:20,209 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:20,801 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:21,805 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:22,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:23,664 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:23,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:24,211 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:24,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:25,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:26,794 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:27,156 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:57:27,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:27,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:27,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:28,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:29,588 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:29,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:30,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:31,795 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:32,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:33,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:34,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:35,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:35,796 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:36,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:37,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:38,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:39,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:40,555 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:40,797 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:41,798 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:42,530 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:42,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:42,801 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:43,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:44,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:45,659 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:45,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:46,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:47,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:48,800 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:49,838 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:50,221 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:50,515 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:57:50,515 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 with policy now +2025-01-07 22:57:50,524 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:57:50,525 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:57:50,525 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:57:50,526 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:57:50,772 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/lyi72qj3-media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:50,816 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:51,221 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:57:51,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:51,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:57:51,511 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/2u0zbof9-media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 22:57:51,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:51,823 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:52,222 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:52,830 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:53,815 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:54,817 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:55,836 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:56,223 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:56,830 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:57,118 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:57:57,156 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:57:57,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:57:57,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:57:57,831 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:58,224 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:57:58,847 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:57:59,850 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:00,843 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:01,846 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:02,192 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:02,193 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:02,193 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:02,193 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:02,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:02,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:02,845 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:03,843 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:04,226 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:04,857 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:05,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:06,851 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:07,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:07,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:08,228 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:08,874 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:09,868 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:10,877 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:11,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:12,229 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:12,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:12,531 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:12,642 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:12,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:13,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:14,865 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:15,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:16,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:17,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:18,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:18,852 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:19,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:20,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:21,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:22,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:23,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:23,853 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:24,854 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:25,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:26,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:27,157 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:58:27,531 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:27,532 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:27,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:28,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:28,855 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:29,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:30,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:31,856 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:32,861 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:33,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:34,559 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:34,869 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:35,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:36,864 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:37,870 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:38,877 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:39,553 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:58:39,553 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 with policy now +2025-01-07 22:58:39,556 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:58:39,565 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:39,566 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:39,566 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:39,566 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:39,842 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/0jqf5dag-media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:39,882 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:40,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:58:40,565 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gisqoycc-media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 22:58:40,875 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:41,906 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:42,241 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:42,564 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:42,564 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:42,909 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:43,915 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:44,242 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:44,903 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:45,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:45,948 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:46,904 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:47,904 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:48,244 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:48,911 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:49,910 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:50,579 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:50,927 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:51,925 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:52,930 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:53,930 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:54,937 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:55,939 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:56,217 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:58:56,247 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:56,254 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:58:56,254 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:58:56,255 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:58:56,957 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:57,157 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:58:57,247 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:58:57,248 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:58:57,536 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:58:57,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:58:57,940 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:58,248 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:58:58,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:58:59,966 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:00,985 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:01,397 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:01,966 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:02,249 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:02,962 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:03,962 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:04,970 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:05,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:06,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:07,230 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:07,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:08,252 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:08,950 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:09,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:10,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:11,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:12,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:12,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:12,621 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:12,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:13,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:14,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:15,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:16,255 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:16,951 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:17,952 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:18,571 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:18,957 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:19,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:20,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:21,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:22,257 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:22,958 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:23,578 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:23,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:24,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:25,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:26,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:27,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:59:27,535 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:27,536 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:27,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:28,641 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:28,959 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:29,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:30,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:31,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:32,960 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:33,828 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:33,977 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:34,262 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:34,664 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 22:59:34,664 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 with policy now +2025-01-07 22:59:34,681 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 22:59:34,681 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:59:34,681 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:59:34,682 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:59:34,918 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/eh83y9ua-media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:34,965 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:35,262 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:59:35,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:35,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 22:59:35,557 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/z1e4jqc8-media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 22:59:35,979 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:36,263 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:36,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:37,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:38,994 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:39,309 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:39,995 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:40,265 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:40,997 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:41,980 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:42,266 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:42,561 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:42,561 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:42,983 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:43,996 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:44,399 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:45,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:45,267 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:59:45,330 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 22:59:45,330 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 22:59:45,330 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 22:59:45,573 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xrrocyx2-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 22:59:46,018 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:46,268 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:46,268 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 22:59:47,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:48,022 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:49,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:49,899 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:50,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:50,270 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 22:59:51,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:52,000 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:53,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:54,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:55,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:55,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 22:59:56,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:57,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:57,158 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 22:59:57,538 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 22:59:57,538 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 22:59:58,001 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 22:59:59,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:00,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:00,593 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:01,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:02,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:03,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:04,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:05,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:06,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:06,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:07,002 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:08,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:09,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:10,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:11,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:12,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:12,539 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:12,539 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:12,687 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:13,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:14,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:15,003 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:16,010 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:16,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:17,012 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:17,150 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:00:17,150 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 with policy now +2025-01-07 23:00:17,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:00:17,153 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:00:17,153 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:00:17,153 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:17,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:00:17,409 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/tqbwf0se-media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:17,685 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/1kylfdea-media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:00:18,028 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:18,281 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:18,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:19,016 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:20,018 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:21,021 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:22,020 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:22,283 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:23,037 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:23,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:24,014 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:24,284 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:25,019 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:26,030 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:27,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:27,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:00:27,573 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:27,573 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:28,034 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:28,718 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:29,022 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:30,032 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:30,286 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:00:30,386 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:00:30,386 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:00:30,387 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:00:31,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:31,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:00:31,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:00:32,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:32,287 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:33,029 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:34,030 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:34,305 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:35,040 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:36,048 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:36,289 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:37,055 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:38,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:39,055 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:39,625 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:40,051 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:41,064 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:42,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:42,291 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:00:42,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:42,547 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:43,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:44,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:44,697 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:45,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:46,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:47,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:48,044 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:49,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:50,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:50,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:51,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:52,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:53,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:54,045 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:55,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:56,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:56,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:00:57,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:57,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:00:57,547 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:00:57,547 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:00:58,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:00:59,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:00,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:01,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:01,672 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:02,046 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:03,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:04,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:05,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:06,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:07,047 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:07,604 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:08,081 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:09,086 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:09,651 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:01:09,651 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 with policy now +2025-01-07 23:01:09,656 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:01:09,657 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:01:09,657 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:01:09,658 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:01:09,902 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/pued8wpr-media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:10,067 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:10,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:01:10,585 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4zdrllnr-media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:01:11,086 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:12,094 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:12,304 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:12,574 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:12,574 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:12,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:13,085 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:14,071 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:14,305 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:15,075 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:16,072 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:17,081 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:18,093 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:18,307 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:18,610 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:19,078 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:20,090 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:21,092 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:22,097 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:23,095 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:23,631 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:24,093 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:25,114 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:26,104 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:27,123 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:27,159 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:01:27,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:27,582 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:28,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:28,645 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:29,129 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:30,128 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:30,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:01:30,459 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:01:30,459 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:01:30,459 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:01:31,121 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:31,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:31,312 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:01:32,133 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:33,170 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:33,313 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:33,857 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:34,126 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:35,131 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:35,314 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:36,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:37,165 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:38,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:39,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:39,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:40,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:41,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:41,317 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:42,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:42,550 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:42,551 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:43,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:43,318 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:44,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:44,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:45,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:46,140 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:47,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:47,319 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:01:48,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:49,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:49,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:50,148 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:51,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:52,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:53,149 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:54,153 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:55,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:55,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:01:56,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:57,151 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:57,160 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:01:57,551 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:01:57,551 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:01:58,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:01:59,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:00,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:00,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:01,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:02,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:03,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:04,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:05,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:06,152 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:06,635 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:07,174 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:08,183 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:08,253 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:02:08,253 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:02:08,254 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 with policy now +2025-01-07 23:02:08,254 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:02:08,254 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:02:08,259 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:08,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:02:08,497 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/be9yfc7s-media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:08,768 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mzfux2k8-media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:02:09,206 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:09,328 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:10,186 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:10,329 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:11,204 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:11,329 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:11,909 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:12,213 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:12,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:12,568 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:13,196 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:13,330 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:14,194 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:15,185 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:16,214 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:17,077 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:17,210 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:17,332 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:18,206 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:19,210 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:20,221 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:21,209 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:22,222 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:22,668 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:23,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:24,220 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:24,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:02:24,520 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:02:24,521 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:02:24,521 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:02:24,592 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/k386vs5c-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:02:25,216 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:25,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:25,335 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:02:26,226 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:27,160 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:02:27,237 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:27,336 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:27,579 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:27,580 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:27,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:28,232 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:29,242 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:30,242 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:31,241 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:31,338 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:32,251 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:33,256 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:33,689 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:34,252 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:35,257 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:36,283 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:37,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:37,340 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:38,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:39,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:39,341 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:39,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:40,247 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:41,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:42,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:42,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:42,568 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:43,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:44,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:44,715 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:45,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:46,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:47,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:48,248 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:49,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:50,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:50,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:51,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:52,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:53,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:53,347 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:02:54,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:55,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:55,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:02:56,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:57,161 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:02:57,249 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:57,568 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:02:57,569 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:02:58,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:02:59,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:00,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:01,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:01,678 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:02,250 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:03,285 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:04,273 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:04,593 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:03:04,594 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 with policy now +2025-01-07 23:03:04,600 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:03:04,600 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:04,600 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:04,607 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:04,864 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/zsgq8r8n-media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:05,260 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:05,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:03:05,642 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/bntajr09-media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:03:06,275 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:06,352 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:06,728 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:07,286 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:07,353 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:08,291 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:09,298 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:10,335 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:11,289 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:11,354 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:12,299 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:12,541 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:12,572 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:12,572 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:13,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:13,355 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:14,318 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:15,307 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:16,314 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:17,297 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:17,707 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:18,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:18,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:03:18,586 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:18,586 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:18,586 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:19,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:19,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:19,358 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:20,297 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:21,305 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:21,359 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:22,296 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:23,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:23,703 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:24,318 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:25,304 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:25,360 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:26,312 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:27,161 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:03:27,317 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:27,594 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:27,595 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:28,327 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:28,709 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:29,344 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:30,355 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:31,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:31,363 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:32,319 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:33,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:33,364 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:34,320 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:34,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:35,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:36,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:37,321 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:38,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:39,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:40,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:40,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:41,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:42,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:42,587 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:42,588 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:43,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:44,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:45,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:45,711 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:46,322 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:47,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:48,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:49,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:50,323 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:51,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:51,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:52,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:53,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:54,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:55,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:56,324 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:56,696 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:03:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:03:57,325 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:57,605 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:03:57,605 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:03:58,348 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:58,531 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:03:58,531 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 with policy now +2025-01-07 23:03:58,534 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:03:58,534 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:03:58,534 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:03:58,535 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:03:58,792 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/7shd7hdg-media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:03:59,361 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:03:59,380 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:03:59,380 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:03:59,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:03:59,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:03:59,678 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/4om8pg02-media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:04:00,335 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:00,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:01,341 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:01,381 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:02,254 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:02,371 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:03,353 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:04,358 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:05,383 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:05,385 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:06,378 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:07,367 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:07,384 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:07,581 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:08,359 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:09,381 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:10,358 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:11,380 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:12,356 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:12,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:04:12,599 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:12,599 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:12,712 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:04:12,712 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:04:12,712 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:12,713 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:04:13,375 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:13,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:13,386 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:04:14,407 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:15,387 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:15,436 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:16,385 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:17,389 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:17,766 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:18,392 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:19,389 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:19,390 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:20,395 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:21,396 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:22,392 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:23,413 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:23,760 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:24,409 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:25,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:26,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:27,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:04:27,392 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:27,403 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:27,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:27,589 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:28,404 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:28,888 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:29,405 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:30,411 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:31,394 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:31,415 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:32,418 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:33,395 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:33,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:34,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:34,743 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:35,421 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:36,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:37,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:38,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:39,422 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:40,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:40,725 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:41,398 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:41,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:42,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:42,589 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:42,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:43,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:44,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:45,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:45,913 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:46,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:47,401 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:47,423 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:48,424 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:49,424 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:50,431 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:51,431 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:51,744 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:52,440 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:53,403 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:53,465 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:53,648 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:04:53,649 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 with policy now +2025-01-07 23:04:53,652 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:04:53,652 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:04:53,653 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:04:53,653 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:04:53,895 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/xiilgsy3-media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:54,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:04:54,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:54,775 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/b8s9beoi-media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:04:55,404 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:55,446 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:56,405 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:56,472 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:57,162 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:04:57,163 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:04:57,446 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:57,605 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:04:57,605 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:04:58,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:04:59,406 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:04:59,473 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:00,475 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:01,477 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:02,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:02,661 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:03,408 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:03,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:04,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:04,482 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:04,715 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:04,715 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:04,716 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:05,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:05,409 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:05,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:06,499 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:07,410 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:07,474 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:07,802 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:08,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:09,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:10,466 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:11,412 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:11,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:12,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:12,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:12,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:13,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:13,776 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:14,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:15,467 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:16,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:17,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:18,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:19,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:19,776 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:20,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:21,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:22,468 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:23,417 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:23,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:24,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:25,418 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:25,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:25,633 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:26,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:27,163 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:05:27,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:27,590 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:27,590 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:28,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:29,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:30,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:30,779 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:31,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:32,469 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:33,470 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:34,470 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:35,488 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:36,328 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:36,475 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:37,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:37,490 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:38,143 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:38,143 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 with policy now +2025-01-07 23:05:38,146 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:38,147 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:38,147 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:38,147 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:38,403 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/i4axdrst-media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:38,424 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:38,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval +2025-01-07 23:05:38,478 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:38,689 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/fvf77b99-media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:05:39,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:39,484 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:40,425 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:40,512 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:41,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:41,799 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:42,491 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:42,613 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:42,614 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:43,426 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:43,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:44,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:45,507 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:46,516 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:46,981 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:47,428 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:47,515 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:48,506 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:49,429 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:49,513 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:49,689 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/567ghusu-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:49,770 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:49,771 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:49,771 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:50,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:50,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:50,562 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:05:50,562 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:05:50,878 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:51,067 INFO wandb-upload_1:2246486 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpanrkz_f3 +2025-01-07 23:05:51,350 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 23:05:51,359 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 23:05:51,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:51,430 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table +2025-01-07 23:05:51,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval +2025-01-07 23:05:51,482 INFO SenderThread:2246486 [sender.py:send_request_log_artifact():1525] logged artifact run-bmk18y9z-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjYxMzY1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkxNTcwMA==', 'latestArtifact': None}} +2025-01-07 23:05:51,482 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:51,482 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json with policy now +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,484 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send():379] send: telemetry +2025-01-07 23:05:51,484 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,485 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,485 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,485 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,486 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,486 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,486 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,487 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,487 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,487 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,487 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,488 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,488 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,488 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,489 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,489 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,489 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,489 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,490 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,490 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,490 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:51,490 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:51,491 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 23:05:51,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:51,664 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/uxopy9e7-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:51,717 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/mphxn0r0-media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:51,891 INFO wandb-upload_0:2246486 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0wk230bn +2025-01-07 23:05:51,930 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/cw8ac4bk-media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:05:52,288 INFO SenderThread:2246486 [sender.py:send_request_log_artifact():1525] logged artifact run-bmk18y9z-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjYxMzY3OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTkxNTcwOQ==', 'latestArtifact': None}} +2025-01-07 23:05:52,288 DEBUG SenderThread:2246486 [sender.py:send():379] send: files +2025-01-07 23:05:52,288 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json with policy now +2025-01-07 23:05:52,289 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:05:52,289 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:52,289 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:05:52,289 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:05:52,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:05:52,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:52,544 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/gyqkjwnx-media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:05:53,431 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:05:53,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:54,500 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:55,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:56,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:57,170 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:05:57,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:57,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:05:57,597 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:05:57,753 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:05:58,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:05:59,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:00,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:01,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:02,501 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:02,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:03,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:04,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:05,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:06,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:07,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:08,502 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:08,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:09,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:10,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:11,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:12,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:12,597 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 23:06:12,598 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: stop_status +2025-01-07 23:06:13,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:13,784 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:14,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:15,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:16,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:17,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:18,503 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:19,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:19,787 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:20,445 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 23:06:20,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:21,504 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:21,581 DEBUG SenderThread:2246486 [sender.py:send():379] send: exit +2025-01-07 23:06:21,581 INFO SenderThread:2246486 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:send_exit():588] handling runtime: 1014 +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:21,582 INFO SenderThread:2246486 [sender.py:send_exit():594] send defer +2025-01-07 23:06:21,582 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,582 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 23:06:21,582 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 1 +2025-01-07 23:06:21,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 23:06:21,583 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 23:06:21,583 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 2 +2025-01-07 23:06:21,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 23:06:21,583 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 23:06:21,584 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 23:06:21,584 DEBUG SystemMonitor:2246486 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 23:06:21,635 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 23:06:21,636 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 23:06:21,636 INFO HandlerThread:2246486 [interfaces.py:finish():200] Joined network monitor +2025-01-07 23:06:21,636 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,636 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 23:06:21,636 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 3 +2025-01-07 23:06:21,636 DEBUG SenderThread:2246486 [sender.py:send():379] send: stats +2025-01-07 23:06:21,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,637 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 23:06:21,637 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:21,637 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 23:06:21,637 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 4 +2025-01-07 23:06:21,637 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:21,637 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 23:06:22,446 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:23,447 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:27,780 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 23:06:27,781 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,781 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 23:06:27,781 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 5 +2025-01-07 23:06:27,781 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:27,782 DEBUG SenderThread:2246486 [sender.py:send():379] send: history +2025-01-07 23:06:27,782 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: summary_record +2025-01-07 23:06:27,782 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,783 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 23:06:27,783 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:27,784 DEBUG SenderThread:2246486 [sender.py:send():379] send: summary +2025-01-07 23:06:27,787 INFO SenderThread:2246486 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 23:06:27,787 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,787 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 6 +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,788 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 23:06:27,788 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 7 +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 23:06:27,788 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:27,788 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 23:06:27,788 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:27,788 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 23:06:28,449 INFO Thread-12 :2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:28,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:30,213 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 8 +2025-01-07 23:06:30,214 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:30,214 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,214 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 23:06:30,214 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,214 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 23:06:30,214 INFO SenderThread:2246486 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 23:06:30,217 INFO SenderThread:2246486 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 23:06:30,219 INFO SenderThread:2246486 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 23:06:30,227 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 9 +2025-01-07 23:06:30,228 DEBUG SenderThread:2246486 [sender.py:send():379] send: artifact +2025-01-07 23:06:30,230 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,231 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjU1MzUzMw==', 'versionIndex': 439}}} +2025-01-07 23:06:30,430 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 23:06:30,430 INFO SenderThread:2246486 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml config.yaml +2025-01-07 23:06:30,450 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json wandb-summary.json +2025-01-07 23:06:30,451 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt requirements.txt +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-metadata.json wandb-metadata.json +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log output.log +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/diff.patch diff.patch +2025-01-07 23:06:30,454 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 23:06:30,458 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 media/videos/eval/12_eval_12_3a187eeb09299696aa2a.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 media/videos/eval/16_eval_16_db31934236335d9e75e4.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 media/videos/eval/1_eval_1_6105fa800baa66794d18.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 media/videos/eval/11_eval_11_a606d9c48c91655df31c.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 media/videos/eval/6_eval_6_a40bb3e24616a11f81cf.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 media/videos/eval/3_eval_3_4978aa6435b32ecf6734.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 media/videos/eval/7_eval_7_1822051540a8ae17e315.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 media/videos/eval/13_eval_13_9efca5d983ee670cf985.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 media/videos/eval/9_eval_9_ae51d826a95728d28cae.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 media/videos/eval/17_eval_17_a0293b276af8b7ee82ad.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 media/videos/eval/5_eval_5_c452dd651b0df3180638.mp4 +2025-01-07 23:06:30,459 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 media/videos/eval/0_eval_0_8a7acf7610c7a7f0d62e.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 media/videos/eval/14_eval_14_ffa92cd8767b9ad59d68.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 media/videos/eval/4_eval_4_ecf6c04acd951daf42e0.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 media/videos/eval/19_eval_19_27b3cf56f899624bcd38.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 media/videos/eval/10_eval_10_1c5570816ad9da40a43e.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 media/videos/eval/8_eval_8_649fa438a50ad92ff033.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 media/videos/eval/15_eval_15_af536578f4476ce23195.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 media/videos/eval/2_eval_2_c92292561966baf2ee38.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 media/videos/eval/18_eval_18_205c297dae90682b6f42.mp4 +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json media/table/eval/success_rate_ci_20_97faedb494e69b73017e.table.json +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json media/table/eval/total_success_rate_ci_21_0e761adafea3a3b09736.table.json +2025-01-07 23:06:30,460 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 10 +2025-01-07 23:06:30,461 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,461 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 23:06:30,461 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,461 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 23:06:30,461 INFO SenderThread:2246486 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:06:30,583 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:30,584 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:30,630 INFO wandb-upload_0:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/wandb-summary.json +2025-01-07 23:06:30,647 INFO wandb-upload_1:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/config.yaml +2025-01-07 23:06:30,720 INFO wandb-upload_2:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/requirements.txt +2025-01-07 23:06:30,724 INFO wandb-upload_4:2246486 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/files/output.log +2025-01-07 23:06:30,733 INFO wandb-upload_3:2246486 [upload_job.py:push():130] Uploaded file /tmp/tmp7pby6zsfwandb/9hmlgza3-events.out.tfevents.1736308167.leonmkim-ROG-Strix-G15CS-G15CS.2246319.0 +2025-01-07 23:06:30,934 INFO Thread-11 (_thread_body):2246486 [sender.py:transition_state():614] send defer: 11 +2025-01-07 23:06:30,934 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,934 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 23:06:30,934 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,935 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 23:06:30,935 INFO SenderThread:2246486 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 12 +2025-01-07 23:06:30,938 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:30,938 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 23:06:30,938 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 23:06:30,938 INFO SenderThread:2246486 [file_stream.py:finish():601] file stream finish called +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [file_stream.py:finish():605] file stream finish is done +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 13 +2025-01-07 23:06:31,115 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:31,115 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 23:06:31,115 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 23:06:31,115 INFO SenderThread:2246486 [sender.py:transition_state():614] send defer: 14 +2025-01-07 23:06:31,115 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: defer +2025-01-07 23:06:31,115 INFO HandlerThread:2246486 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send():379] send: final +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send():379] send: footer +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: defer +2025-01-07 23:06:31,116 INFO SenderThread:2246486 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 23:06:31,116 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:31,116 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 23:06:31,117 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 23:06:31,118 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 23:06:31,118 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: server_info +2025-01-07 23:06:31,182 DEBUG SenderThread:2246486 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 23:06:31,182 INFO MainThread:2246486 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 23:06:31,183 INFO MainThread:2246486 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 23:06:31,183 INFO MainThread:2246486 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 23:06:31,184 DEBUG HandlerThread:2246486 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 23:06:31,184 INFO HandlerThread:2246486 [handler.py:finish():882] shutting down handler +2025-01-07 23:06:32,118 INFO WriterThread:2246486 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [sender.py:finish():1615] shutting down sender +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 23:06:32,182 INFO SenderThread:2246486 [file_pusher.py:join():175] waiting for file pusher diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug.log b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70464a8cb3f0d2b819fefb05459df734a59e8e6d --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 22:49:26,770 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 22:49:26,770 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Configure stats pid to 2246319 +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/settings +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug.log +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/logs/debug-internal.log +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():569] calling init triggers +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1002_0', 'wandb_notes': '1002_0_req_1365_0', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 20, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z'} +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():619] starting backend +2025-01-07 22:49:26,771 INFO MainThread:2246319 [wandb_init.py:init():623] setting up manager +2025-01-07 22:49:26,775 INFO MainThread:2246319 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 22:49:26,777 INFO MainThread:2246319 [wandb_init.py:init():631] backend started and connected +2025-01-07 22:49:26,788 INFO MainThread:2246319 [wandb_init.py:init():720] updated telemetry +2025-01-07 22:49:26,795 INFO MainThread:2246319 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 22:49:27,053 INFO MainThread:2246319 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 22:49:27,093 INFO MainThread:2246319 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 22:49:27,093 INFO MainThread:2246319 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 22:49:27,506 INFO MainThread:2246319 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 22:49:27,506 INFO MainThread:2246319 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 22:49:27,507 INFO MainThread:2246319 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 22:49:27,507 INFO MainThread:2246319 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 22:49:27,508 INFO MainThread:2246319 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 22:49:27,509 INFO MainThread:2246319 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1002_0/bmk18y9z/tb, True +2025-01-07 22:49:31,314 INFO MainThread:2246319 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-07 23:06:32,184 WARNING MsgRouterThr:2246319 [router.py:message_loop():77] message_loop has been closed diff --git a/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e9e2ee9b351d05c981bff91119e957180792dcdf --- /dev/null +++ b/bmk18y9z/wandb/run-20250107_224926-bmk18y9z/run-bmk18y9z.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:856e90f08f3cbf8817e12187fe1ef5eb49381c4832380ec0fee385c2d93c9849 +size 239402 diff --git a/jqiikotd/episode_rosbags/episode_1_2025-01-07-17-24-24.bag b/jqiikotd/episode_rosbags/episode_1_2025-01-07-17-24-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..af3592f1beaccfc2af2062d0c6f4cc86da2dc24d --- /dev/null +++ b/jqiikotd/episode_rosbags/episode_1_2025-01-07-17-24-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ae6530aaf704e897264e038c081a24ac905ed147951f83857e070ad99b48bd +size 1418890866 diff --git a/rgxendfg/episode_rosbags/episode_1_2025-01-07-20-55-45.bag b/rgxendfg/episode_rosbags/episode_1_2025-01-07-20-55-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..640b963b9403d8157c072070c6a73d2dbbe14167 --- /dev/null +++ b/rgxendfg/episode_rosbags/episode_1_2025-01-07-20-55-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bff405e607c863e51ef64f7bb572d8cf9c9b5b90c36a7c41df84eef2bc28f79 +size 1411794742 diff --git a/rgxendfg/episode_rosbags/episode_2_2025-01-07-20-56-31.bag b/rgxendfg/episode_rosbags/episode_2_2025-01-07-20-56-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..ab6972b0752339918601939d9a13082d5ba86f19 --- /dev/null +++ b/rgxendfg/episode_rosbags/episode_2_2025-01-07-20-56-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:400cbbca078a4cbec0ff101f13debf6238127e4146fc11b5becba58bb3a9b433 +size 1436947352 diff --git a/rgxendfg/episode_rosbags/episode_3_2025-01-07-20-57-21.bag b/rgxendfg/episode_rosbags/episode_3_2025-01-07-20-57-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..d749f946ec010086a3996cd47f1d765f520c4c77 --- /dev/null +++ b/rgxendfg/episode_rosbags/episode_3_2025-01-07-20-57-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:176d071fae0963df9782bd8e3400d79191207db0731385b04cf285648fec920e +size 1436284371 diff --git a/z6f2akbj/episode_rosbags/episode_0_2025-01-07-20-59-59.bag b/z6f2akbj/episode_rosbags/episode_0_2025-01-07-20-59-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..e15677750b998e5a0982e99610303037432cb8ec --- /dev/null +++ b/z6f2akbj/episode_rosbags/episode_0_2025-01-07-20-59-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13a470b0058de6c4295f5c0933c335e1505518b3ce0f9b392272ce69411ec8f +size 1411903426 diff --git a/z6f2akbj/episode_rosbags/episode_1_2025-01-07-21-00-47.bag b/z6f2akbj/episode_rosbags/episode_1_2025-01-07-21-00-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..a3a0fd966a96cdac6bf17500887c2564119ebae8 --- /dev/null +++ b/z6f2akbj/episode_rosbags/episode_1_2025-01-07-21-00-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35bdaeba27b5648d446968628b9beec0e1a0c4ba2333ad630ba7c7fd1f330316 +size 1410631886 diff --git a/z6f2akbj/episode_rosbags/episode_2_2025-01-07-21-01-29.bag b/z6f2akbj/episode_rosbags/episode_2_2025-01-07-21-01-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..b400cbafb1d8a31ac46ad5eef13d5a3b1bff718a --- /dev/null +++ b/z6f2akbj/episode_rosbags/episode_2_2025-01-07-21-01-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b51ceac094e106b3e434019d299da4f18e550eacde811f1bb53eca79b13b56c2 +size 1413045664 diff --git a/z6f2akbj/episode_rosbags/episode_3_2025-01-07-21-02-51.bag b/z6f2akbj/episode_rosbags/episode_3_2025-01-07-21-02-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..0b5be498882447be7b5851467d1b7345cc3c2b7f --- /dev/null +++ b/z6f2akbj/episode_rosbags/episode_3_2025-01-07-21-02-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279c6bd876995ae714efd01062369ce26e1759c5c585d93e34fe054b5567aa5f +size 1038196023